Merge branch 'stretch-unstable' into improve-service-status-checks

data/actionsmap/yunohost.yml

@@ -543,38 +543,6 @@ app:
     category_help: Manage apps
     actions:
 
-        ### app_fetchlist()
-        fetchlist:
-            action_help: Fetch application lists from app servers, or register a new one.
-            api: PUT /appslists
-            arguments:
-                -n:
-                    full: --name
-                    help: Name of the list to fetch (fetches all registered lists if empty)
-                    extra:
-                        pattern: &pattern_listname
-                            - !!str ^[a-z0-9_]+$
-                            - "pattern_listname"
-                -u:
-                    full: --url
-                    help: URL of a new application list to register. To be specified with -n.
-
-        ### app_listlists()
-        listlists:
-            action_help: List registered application lists
-            api: GET /appslists
-
-        ### app_removelist()
-        removelist:
-            action_help: Remove and forget about a given application list
-            api: DELETE /appslists
-            arguments:
-                name:
-                    help: Name of the list to remove
-                    extra:
-                        ask: ask_list_to_remove
-                        pattern: *pattern_listname
-
         ### app_list()
         list:
             action_help: List apps

data/helpers.d/network

@@ -17,7 +17,7 @@ ynh_find_port () {
 	ynh_handle_getopts_args "$@"
 
 	test -n "$port" || ynh_die --message="The argument of ynh_find_port must be a valid port."
-	while netcat -z 127.0.0.1 $port       # Check if the port is free
+	while ss -nltu | grep -q -w :$port       # Check if the port is free
 	do
 		port=$((port+1))	# Else, pass to next port
 	done

data/hooks/diagnosis/14-ports.py

@@ -26,7 +26,7 @@ class PortsDiagnoser(Diagnoser):
                 ports[port] = service
 
         try:
-            r = requests.post('https://ynhdiagnoser.netlib.re/check-ports', json={'ports': ports.keys()}, timeout=30).json()
+            r = requests.post('https://diagnosis.yunohost.org/check-ports', json={'ports': ports.keys()}, timeout=30).json()
             if "status" not in r.keys():
                 raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
             elif r["status"] == "error":

data/hooks/diagnosis/16-http.py

@@ -28,7 +28,7 @@ class HttpDiagnoser(Diagnoser):
             os.system("touch /tmp/.well-known/ynh-diagnosis/%s" % nonce)
 
             try:
-                r = requests.post('https://ynhdiagnoser.netlib.re/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json()
+                r = requests.post('https://diagnosis.yunohost.org/check-http', json={'domain': domain, "nonce": nonce}, timeout=30).json()
                 if "status" not in r.keys():
                     raise Exception("Bad syntax for response ? Raw json: %s" % str(r))
                 elif r["status"] == "error" and ("code" not in r.keys() or r["code"] not in ["error_http_check_connection_error", "error_http_check_unknown_error"]):

locales/ca.json

@@ -162,7 +162,7 @@
     "admin_password_too_long": "Trieu una contrasenya de menys de 127 caràcters",
     "dpkg_is_broken": "No es pot fer això en aquest instant perquè dpkg/APT (els gestors de paquets del sistema) sembla estar mal configurat… Podeu intentar solucionar-ho connectant-vos per SSH i executant «sudo dpkg --configure -a».",
     "dnsmasq_isnt_installed": "sembla que dnsmasq no està instal·lat, executeu \"apt-get remove bind9 && apt-get install dnsmasq\"",
-    "domain_cannot_remove_main": "No es pot eliminar el domini principal. S'ha d'establir un nou domini primer",
+    "domain_cannot_remove_main": "No es pot eliminar «{domain:s}» ja que és el domini principal, primer s'ha d'establir un nou domini principal utilitzant «yunohost domain main-domain -n <un-altre-domini>», aquí hi ha una llista dels possibles dominis: {other_domains:s}",
     "domain_cert_gen_failed": "No s'ha pogut generar el certificat",
     "domain_created": "S'ha creat el domini",
     "domain_creation_failed": "No s'ha pogut crear el domini {domain}: {error}",
@@ -308,7 +308,7 @@
     "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "La configuració SSH serà gestionada per YunoHost (pas 1, automàtic)",
     "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "La configuració SSH serà gestionada per YunoHost (pas 2, manual)",
     "migration_description_0009_decouple_regenconf_from_services": "Desvincula el mecanisme regen-conf dels serveis",
-    "migration_description_0010_migrate_to_apps_json": "Elimina les llistes d'aplicacions obsoletes i utilitza la nova llista unificada «apps.json» en el seu lloc",
+    "migration_description_0010_migrate_to_apps_json": "Elimina els catàlegs d'aplicacions obsolets i utilitza la nova llista unificada «apps.json» en el seu lloc (obsolet, substituït per la migració 13)",
     "migration_0003_backward_impossible": "La migració Stretch no és reversible.",
     "migration_0003_start": "Ha començat la migració a Stretch. Els registres estaran disponibles a {logfile}.",
     "migration_0003_patching_sources_list": "Modificant el fitxer sources.lists…",
@@ -320,7 +320,7 @@
     "migration_0003_system_not_fully_up_to_date": "El vostre sistema no està completament actualitzat. S'ha de fer una actualització normal abans de fer la migració a Stretch.",
     "migration_0003_still_on_jessie_after_main_upgrade": "Hi ha hagut un problema durant l'actualització principal: El sistema encara està amb Jessie? Per investigar el problema, mireu el registres a {log}:s…",
     "migration_0003_general_warning": "Tingueu en compte que la migració és una operació delicada. L'equip de YunoHost a fet els possibles per revisar-la i provar-la, però la migració pot provocar errors en parts del sistema o aplicacions.\n\nPer tant, es recomana:\n   - Fer una còpia de seguretat de les dades o aplicacions importants. Més informació a https://yunohost.org/backup;\n   - Sigueu pacient un cop llençada la migració: en funció de la connexió a internet i el maquinari, pot trigar fins a unes hores per actualitzar-ho tot.\n\nD'altra banda, el port per SMTP, utilitzat per clients de correu externs (com Thunderbird o K9-Mail) ha canviat de 465 (SSL/TLS) a 587 (STARTTLS). L'antic port (465) serà tancat automàticament, i el nou port (587) serà obert en el tallafocs. Tots els usuaris *hauran* d'adaptar la configuració dels clients de correu en acord amb aquests canvis.",
-    "migration_0003_problematic_apps_warning": "Tingueu en compte que s'han detectat les aplicacions, possiblement, problemàtiques següents. Sembla que aquestes no s'han instal·lat des d'una applist, o que no estan marcades com a «working». Per conseqüent, no podem garantir que segueixin funcionant després de l'actualització: {problematic_apps}",
+    "migration_0003_problematic_apps_warning": "Tingueu en compte que s'han detectat les aplicacions, possiblement, problemàtiques següents. Sembla que aquestes no s'han instal·lat des d'un catàleg  d'aplicacions, o que no estan marcades com a «working». Per conseqüent, no podem garantir que segueixin funcionant després de l'actualització: {problematic_apps}",
     "migration_0003_modified_files": "Tingueu en compte que s'han detectat els següents fitxers que han estat modificats manualment i podrien sobreescriure's al final de l'actualització: {manually_modified_files}",
     "migration_0005_postgresql_94_not_installed": "PostgreSQL no està instal·lat en el sistema. No hi ha res per fer.",
     "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 està instal·lat, però no PostgreSQL 9.6? Alguna cosa estranya a passat en el sistema :( …",
@@ -627,5 +627,81 @@
     "permission_currently_allowed_for_visitors": "El permís ja el tenen el grup de visitants a més d'altres grups. Segurament s'hauria de revocar el permís al grup dels visitants o eliminar els altres grups als que s'ha atribuït.",
     "permission_currently_allowed_for_all_users": "El permís ha el té el grup de tots els usuaris (all_users) a més d'altres grups. Segurament s'hauria de revocar el permís a «all_users» o eliminar els altres grups als que s'ha atribuït.",
     "permission_require_account": "El permís {permission} només té sentit per als usuaris que tenen un compte, i per tant no es pot activar per als visitants.",
-    "app_remove_after_failed_install": "Eliminant l'aplicació després que hagi fallat la instal·lació…"
+    "app_remove_after_failed_install": "Eliminant l'aplicació després que hagi fallat la instal·lació…",
+    "diagnosis_basesystem_ynh_main_version": "El servidor funciona amb YunoHost {main_version} ({repo})",
+    "diagnosis_ram_low": "El sistema només té {available_abs_MB} MB ({available_percent}%) de memòria RAM disponibles d'un total de {total_abs_MB} MB. Aneu amb compte.",
+    "diagnosis_swap_none": "El sistema no té swap. Hauríeu de considerar afegir un mínim de 256 MB de swap per evitar situacions en les que el sistema es queda sense memòria.",
+    "diagnosis_regenconf_manually_modified": "El fitxer de configuració {file} ha estat modificat manualment.",
+    "diagnosis_regenconf_nginx_conf_broken": "Sembla que s'ha trencat la configuració NGINX!",
+    "diagnosis_security_vulnerable_to_meltdown_details": "Per arreglar-ho, hauríeu d'actualitzar i reiniciar el sistema per tal de carregar el nou nucli de linux (o contactar amb el proveïdor del servidor si no funciona). Vegeu https://meltdownattack.com/ per a més informació.",
+    "diagnosis_http_could_not_diagnose": "No s'ha pogut diagnosticar si el domini és accessible des de l'exterior. Error: {error}",
+    "domain_cannot_remove_main_add_new_one": "No es pot eliminar «{domain:s}» ja que és el domini principal i únic domini, primer s'ha d'afegir un altre domini utilitzant «yunohost domain add <un-altre-domini.com>», i després fer-lo el domini principal amb «yunohost domain main-domain -n <un-altre-domini.com>» i després es pot eliminar el domini «{domain:s}» utilitzant «yunohost domain remove {domain:s}».",
+    "diagnosis_basesystem_host": "El servidor funciona amb Debian {debian_version}.",
+    "diagnosis_basesystem_kernel": "El servidor funciona amb el nucli de Linux {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} versió: {1}({2})",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "Esteu utilitzant versions inconsistents dels paquets de YunoHost… probablement a causa d'una actualització fallida o parcial.",
+    "diagnosis_display_tip_web": "Podeu anar a la secció de Diagnòstics (en la pantalla principal) per veure els errors que s'han trobat.",
+    "diagnosis_failed_for_category": "Ha fallat el diagnòstic per la categoria «{category}» : {error}",
+    "diagnosis_display_tip_cli": "Podeu executar «yunohost diagnosis show --issues» per mostrar els errors que s'han trobat.",
+    "diagnosis_cache_still_valid": "(La memòria cau encara és vàlida pel diagnòstic de {category}. No es tornar a diagnosticar de moment!)",
+    "diagnosis_cant_run_because_of_dep": "No es pot fer el diagnòstic per {category} mentre hi ha problemes importants relacionats amb {dep}.",
+    "diagnosis_ignored_issues": "(+ {nb_ignored} problema(es) ignorat(s))",
+    "diagnosis_found_errors": "S'ha trobat problema(es) important(s) {errors} relacionats amb {category}!",
+    "diagnosis_found_errors_and_warnings": "S'ha trobat problema(es) important(s) {errors} (i avis(os) {warnings}) relacionats amb {category}!",
+    "diagnosis_found_warnings": "S'han trobat ítems {warnings} que es podrien millorar per {category}.",
+    "diagnosis_everything_ok": "Tot sembla correcte per {category}!",
+    "diagnosis_failed": "No s'han pogut obtenir els resultats del diagnòstic per la categoria «{category}» : {error}",
+    "diagnosis_ip_connected_ipv4": "El servidor està connectat a Internet amb IPv4!",
+    "diagnosis_ip_no_ipv4": "El servidor no té una IPv4 que funcioni.",
+    "diagnosis_ip_connected_ipv6": "El servidor està connectat a Internet amb IPv6!",
+    "diagnosis_ip_no_ipv6": "El servidor no té una IPv6 que funcioni.",
+    "diagnosis_ip_not_connected_at_all": "Sembla que el servidor no està connectat a internet!?",
+    "diagnosis_ip_dnsresolution_working": "La resolució de nom de domini està funcionant!",
+    "diagnosis_ip_broken_dnsresolution": "La resolució de nom de domini falla per algun motiu… Està el tallafocs bloquejant les peticions DNS?",
+    "diagnosis_ip_broken_resolvconf": "La resolució de nom de domini sembla caiguda en el servidor, podria estar relacionat amb el fet que /etc/resolv.conf no apunta cap a 127.0.0.1.",
+    "diagnosis_ip_weird_resolvconf": "La resolució DNS sembla estar funcionant, però aneu amb compte ja que esteu utilitzant un versió personalitzada de /etc/resolv.conf.",
+    "diagnosis_ip_weird_resolvconf_details": "En canvi, aquest fitxer hauria de ser un enllaç simbòlic cap a /etc/resolvconf/run/resolv.conf i que aquest apunti cap a 127.0.0.1 (dnsmasq). La configuració del «resolver» real s'hauria de fer via /etc/resolv.dnsmaq.conf.",
+    "diagnosis_dns_good_conf": "Bona configuració DNS pel domini {domain} (categoria {category})",
+    "diagnosis_dns_bad_conf": "Configuració DNS incorrecta o inexistent pel domini {domain} (categoria {category})",
+    "diagnosis_dns_missing_record": "Segons la configuració DNS recomanada, hauríeu d'afegir un registre DNS de tipus {0}, nom {1} i valor {2}",
+    "diagnosis_dns_discrepancy": "Segons la configuració DNS recomanada, el valor pel registre DNS de tipus {0} i nom {1} hauria de ser {2}, en comptes de {3}.",
+    "diagnosis_services_good_status": "El servei {service} està {status} tal i com s'esperava!",
+    "diagnosis_services_bad_status": "El servei {service} està {status} :/",
+    "diagnosis_diskusage_verylow": "El lloc d'emmagatzematge {mountpoint} (en l'aparell {device}) només té disponibles {free_abs_GB} GB ({free_percent}%). Hauríeu de considerar alliberar una mica d'espai.",
+    "diagnosis_diskusage_low": "El lloc d'emmagatzematge {mountpoint} (en l'aparell {device}) només té disponibles {free_abs_GB} GB ({free_percent}%). Aneu amb compte.",
+    "diagnosis_diskusage_ok": "El lloc d'emmagatzematge {mountpoint} (en l'aparell {device}) encara té {free_abs_GB} GB ({free_percent}%) lliures!",
+    "diagnosis_ram_verylow": "El sistema només té {available_abs_MB} MB ({available_percent}%) de memòria RAM disponibles! (d'un total de {total_abs_MB} MB)",
+    "diagnosis_ram_ok": "El sistema encara té {available_abs_MB} MB ({available_percent}%) de memòria RAM disponibles d'un total de {total_abs_MB} MB.",
+    "diagnosis_swap_notsomuch": "El sistema només té {total_MB} MB de swap. Hauríeu de considerar tenir un mínim de 256 MB per evitar situacions en les que el sistema es queda sense memòria.",
+    "diagnosis_swap_ok": "El sistema té {total_MB} MB de swap!",
+    "diagnosis_regenconf_allgood": "Tots els fitxers de configuració estan en acord amb la configuració recomanada!",
+    "diagnosis_regenconf_manually_modified_details": "No hauria de ser cap problema sempre i quan sapigueu el que esteu fent ;) !",
+    "diagnosis_regenconf_manually_modified_debian": "El fitxer de configuració {file} ha estat modificat manualment respecte al fitxer per defecte de Debian.",
+    "diagnosis_regenconf_manually_modified_debian_details": "No hauria de ser cap problema, però ho haureu de vigilar...",
+    "diagnosis_security_all_good": "No s'ha trobat cap vulnerabilitat de seguretat crítica.",
+    "diagnosis_security_vulnerable_to_meltdown": "Sembla que el sistema és vulnerable a la vulnerabilitat de seguretat crítica Meltdown",
+    "diagnosis_description_basesystem": "Sistema de base",
+    "diagnosis_description_ip": "Connectivitat a Internet",
+    "diagnosis_description_dnsrecords": "Registres DNS",
+    "diagnosis_description_services": "Verificació de l'estat dels serveis",
+    "diagnosis_description_systemresources": "Recursos del sistema",
+    "diagnosis_description_ports": "Exposició dels ports",
+    "diagnosis_description_http": "Exposició HTTP",
+    "diagnosis_description_regenconf": "Configuració del sistema",
+    "diagnosis_description_security": "Verificacions de seguretat",
+    "diagnosis_ports_could_not_diagnose": "No s'ha pogut diagnosticar si els ports són accessibles des de l'exterior. Error: {error}",
+    "diagnosis_ports_unreachable": "El port {port} no és accessible des de l'exterior.",
+    "diagnosis_ports_ok": "El port {port} és accessible des de l'exterior.",
+    "diagnosis_http_ok": "El domini {domain} és accessible des de l'exterior.",
+    "diagnosis_http_unreachable": "El domini {domain} no és accessible a través de HTTP des de l'exterior.",
+    "diagnosis_unknown_categories": "Les següents categories són desconegudes: {categories}",
+    "apps_catalog_init_success": "S'ha iniciat el sistema de catàleg d'aplicacions!",
+    "apps_catalog_updating": "S'està actualitzant el catàleg d'aplicacions…",
+    "apps_catalog_failed_to_download": "No s'ha pogut descarregar el catàleg d'aplicacions {apps_catalog}: {error}",
+    "apps_catalog_obsolete_cache": "La memòria cau del catàleg d'aplicacions és buida o obsoleta.",
+    "apps_catalog_update_success": "S'ha actualitzat el catàleg d'aplicacions!",
+    "diagnosis_mail_ougoing_port_25_ok": "El port de sortida 25 no està bloquejat i els correus es poden enviar a altres servidors.",
+    "diagnosis_mail_ougoing_port_25_blocked": "Sembla que el port de sortida 25 està bloquejat. Hauríeu d'intentar desbloquejar-lo al panell de configuració del proveïdor d'accés a internet (o allotjador). Mentrestant, el servidor no podrà enviar correus a altres servidors.",
+    "diagnosis_description_mail": "Correu electrònic",
+    "migration_description_0013_futureproof_apps_catalog_system": "Migrar al nou sistema de catàleg d'aplicacions resistent al pas del temps"
 }

locales/en.json

@@ -54,21 +54,16 @@
     "app_upgraded": "{app:s} upgraded",
     "apps_already_up_to_date": "All apps are already up-to-date",
     "apps_permission_not_found": "No permission found for the installed apps",
-    "appslist_corrupted_json": "Could not load the app lists. It looks like {filename:s} is damaged.",
+    "apps_permission_restoration_failed": "Permission '{permission:s}' for app {app:s} restoration has failed",
-    "appslist_could_not_migrate": "Could not migrate the app list '{appslist:s}'! Could not parse the URL… The old cron job was kept kept in {bkp_file:s}.",
+    "apps_catalog_init_success": "Apps catalog system initialized!",
-    "appslist_fetched": "Updated the app list '{appslist:s}'",
+    "apps_catalog_updating": "Updating applications catalog...",
-    "appslist_migrating": "Migrating the app list '{appslist:s}'…",
+    "apps_catalog_failed_to_download": "Unable to download the {apps_catalog} apps catalog: {error}",
-    "appslist_name_already_tracked": "A registered app list with the name {name:s} already exists.",
+    "apps_catalog_obsolete_cache": "The apps catalog cache is empty or obsolete.",
-    "appslist_removed": "The '{appslist:s}' app list was removed",
+    "apps_catalog_update_success": "The application catalog has been updated!",
-    "appslist_retrieve_bad_format": "Could not read the fetched app list '{appslist:s}'",
-    "appslist_retrieve_error": "Cannot retrieve the remote app list '{appslist:s}': {error:s}",
-    "appslist_unknown": "The app list '{appslist:s}' is unknown.",
-    "appslist_url_already_tracked": "There is already a registered app list with the URL {url:s}.",
     "ask_current_admin_password": "Current administration password",
     "ask_email": "E-mail address",
     "ask_firstname": "First name",
     "ask_lastname": "Last name",
-    "ask_list_to_remove": "List to remove",
     "ask_main_domain": "Main domain",
     "ask_new_admin_password": "New administration password",
     "ask_new_domain": "New domain",
@@ -149,7 +144,6 @@
     "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'",
     "confirm_app_install_thirdparty": "DANGER! This app is not part of Yunohost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or break your system… If you are willing to take that risk anyway, type '{answers:s}'",
     "custom_app_url_required": "You must provide a URL to upgrade your custom app {app:s}",
-    "custom_appslist_name_required": "You must provide a name for your custom app list",
     "diagnosis_basesystem_host": "Server is running Debian {debian_version}.",
     "diagnosis_basesystem_kernel": "Server is running Linux kernel {kernel_version}",
     "diagnosis_basesystem_ynh_single_version": "{0} version: {1} ({2})",
@@ -319,8 +313,6 @@
     "log_help_to_get_failed_log": "The operation '{desc}' could not be completed. Please share the full log of this operation using the command 'yunohost log display {name} --share' to get help",
     "log_does_exists": "There is not operation log with the name '{log}', use 'yunohost log list' to see all available operation logs",
     "log_operation_unit_unclosed_properly": "Operation unit has not been closed properly",
-    "log_app_fetchlist": "Add an app list",
-    "log_app_removelist": "Remove an app list",
     "log_app_change_url": "Change the URL of the '{}' app",
     "log_app_install": "Install the '{}' app",
     "log_app_remove": "Remove the '{}' app",
@@ -384,9 +376,10 @@
     "migration_description_0007_ssh_conf_managed_by_yunohost_step1": "Let the SSH configuration be managed by YunoHost (step 1, automatic)",
     "migration_description_0008_ssh_conf_managed_by_yunohost_step2": "Let the SSH configuration be managed by YunoHost (step 2, manual)",
     "migration_description_0009_decouple_regenconf_from_services": "Decouple the regen-conf mechanism from services",
-    "migration_description_0010_migrate_to_apps_json": "Remove deprecated applists and use the new unified 'apps.json' list instead",
+    "migration_description_0010_migrate_to_apps_json": "Remove deprecated apps catalogs and use the new unified 'apps.json' list instead (outdated, replaced by migration 13)",
     "migration_description_0011_setup_group_permission": "Set up user group and set up permission for apps and services",
     "migration_description_0012_postgresql_password_to_md5_authentication": "Force PostgreSQL authentication to use MD5 for local connections",
+    "migration_description_0013_futureproof_apps_catalog_system": "Migrate to the new future-proof apps catalog system",
     "migration_0003_start": "Starting migration to Stretch. The logs will be available in {logfile}.",
     "migration_0003_patching_sources_list": "Patching the sources.lists…",
     "migration_0003_main_upgrade": "Starting main upgrade…",
@@ -397,7 +390,7 @@
     "migration_0003_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Stretch.",
     "migration_0003_still_on_jessie_after_main_upgrade": "Something went wrong during the main upgrade: Is the system still on Jessie‽ To investigate the issue, please look at {log}:s…",
     "migration_0003_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n    - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n    - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.\n\nAdditionally, the port for SMTP, used by external e-mail clients (like Thunderbird or K9-Mail) was changed from 465 (SSL/TLS) to 587 (STARTTLS). The old port (465) will automatically be closed, and the new port (587) will be opened in the firewall. You and your users *will* have to adapt the configuration of your e-mail clients accordingly.",
-    "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an applist, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}",
+    "migration_0003_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from an apps_catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}",
     "migration_0003_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}",
     "migration_0005_postgresql_94_not_installed": "PostgreSQL was not installed on your system. Nothing to do.",
     "migration_0005_postgresql_96_not_installed": "PostgreSQL 9.4 is installed, but not postgresql 9.6‽ Something weird might have happened on your system:(…",

locales/eo.json

@@ -564,5 +564,16 @@
     "permission_currently_allowed_for_all_users": "Ĉi tiu permeso estas nuntempe donita al ĉiuj uzantoj aldone al aliaj grupoj. Vi probable volas aŭ forigi la permeson \"all_users\" aŭ forigi la aliajn grupojn, kiujn ĝi nuntempe donas.",
     "app_install_failed": "Ne povis instali {app} : {error}",
     "app_install_script_failed": "Eraro okazis en la skripto de instalado de la app",
-    "app_remove_after_failed_install": "Forigado de la app post la instala fiasko …"
+    "app_remove_after_failed_install": "Forigado de la app post la instala fiasko …",
+    "diagnosis_basesystem_host": "Servilo funkcias Debian {debian_version}.",
+    "apps_catalog_init_success": "Aplikoj katalogsistemo inicializita !",
+    "apps_catalog_updating": "Ĝisdatigante katalogo de aplikoj ...",
+    "apps_catalog_failed_to_download": "Ne eblas elŝuti la katalogon de {apps_catalog}: {error}",
+    "apps_catalog_obsolete_cache": "La kaŝmemoro de la katalogo de programoj estas malplena aŭ malaktuala.",
+    "apps_catalog_update_success": "La aplika katalogo estis ĝisdatigita!",
+    "diagnosis_basesystem_kernel": "Servilo funkcias Linuksan kernon {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} versio: {1} ({2})",
+    "diagnosis_basesystem_ynh_main_version": "Servilo funkcias YunoHost {main_version} ({repo})",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "Vi prizorgas malkonsekvencajn versiojn de la YunoHost-pakoj... plej probable pro malsukcesa aŭ parta ĝisdatigo.",
+    "diagnosis_display_tip_web": "Vi povas iri al la sekcio Diagnozo (en la hejmekrano) por vidi la trovitajn problemojn."
 }

locales/es.json

@@ -638,5 +638,25 @@
     "permission_currently_allowed_for_visitors": "Este permiso se concede actualmente a los visitantes además de otros grupos. Probablemente quiere o eliminar el permiso de «visitors» o eliminar los otros grupos a los que está otorgado actualmente.",
     "permission_currently_allowed_for_all_users": "Este permiso se concede actualmente a todos los usuarios además de los otros grupos. Probablemente quiere o eliminar el permiso de «all_users» o eliminar los otros grupos a los que está otorgado actualmente.",
     "permission_require_account": "El permiso {permission} solo tiene sentido para usuarios con una cuenta y, por lo tanto, no se puede activar para visitantes.",
-    "app_remove_after_failed_install": "Eliminando la aplicación tras el fallo de instalación…"
+    "app_remove_after_failed_install": "Eliminando la aplicación tras el fallo de instalación…",
+    "diagnosis_basesystem_host": "El servidor está ejecutando Debian {debian_version}.",
+    "diagnosis_basesystem_kernel": "El servidor está ejecutando el núcleo de Linux {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} versión: {1} ({2})",
+    "diagnosis_basesystem_ynh_main_version": "El servidor está ejecutando YunoHost {main_version} ({repo})",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "Está ejecutando versiones incoherentes de los paquetes de YunoHost... probablemente por una actualización errónea o parcial.",
+    "diagnosis_failed_for_category": "Diagnóstico fallido para la categoría «{category}» : {error}",
+    "diagnosis_cache_still_valid": "(Caché aún válida para el diagnóstico de {category}. ¡Aún no se ha rediagnosticado!)",
+    "diagnosis_found_errors_and_warnings": "¡Encontrado(s) error(es) significativo(s) {errors} (y aviso(s) {warnings}) relacionado(s) con {category}!",
+    "diagnosis_display_tip_web": "Puede ir a la sección de diagnóstico (en la pantalla principal)  para ver los problemas encontrados.",
+    "diagnosis_display_tip_cli": "Puede ejecutar «yunohost diagnosis show --issues» para mostrar los problemas encontrados.",
+    "apps_catalog_init_success": "¡Sistema de catálogo de aplicaciones inicializado!",
+    "apps_catalog_updating": "Actualizando catálogo de aplicaciones...",
+    "apps_catalog_failed_to_download": "No se pudo descargar el catálogo de aplicaciones {apps_catalog}: {error}",
+    "apps_catalog_obsolete_cache": "La caché del catálogo de aplicaciones está vacía u obsoleta.",
+    "apps_catalog_update_success": "¡El catálogo de aplicaciones ha sido actualizado!",
+    "diagnosis_cant_run_because_of_dep": "No se puede ejecutar el diagnóstico para {category} mientras haya problemas importantes relacionados con {dep}.",
+    "diagnosis_ignored_issues": "(+ {nb_ignored} problema(s) ignorado(s))",
+    "diagnosis_found_errors": "¡Encontrado(s) error(es) significativo(s) {errors} relacionado(s) con {category}!",
+    "diagnosis_found_warnings": "Encontrado elemento(s) {warnings} que puede(n) ser mejorado(s) para {category}.",
+    "diagnosis_everything_ok": "¡Todo se ve bien para {category}!"
 }

locales/fr.json

@@ -77,7 +77,7 @@
     "domain_created": "Le domaine a été créé",
     "domain_creation_failed": "Impossible de créer le domaine {domain}: {error}",
     "domain_deleted": "Le domaine a été supprimé",
-    "domain_deletion_failed": "Impossible de supprimer le domaine {domain}: {error}",
+    "domain_deletion_failed": "Impossible de supprimer le domaine {domain}:{error}",
     "domain_dyndns_already_subscribed": "Vous avez déjà souscris à un domaine DynDNS",
     "domain_dyndns_invalid": "Domaine incorrect pour un usage avec DynDNS",
     "domain_dyndns_root_unknown": "Domaine DynDNS principal inconnu",
@@ -270,7 +270,7 @@
     "ldap_init_failed_to_create_admin": "L’initialisation de l'annuaire LDAP n’a pas réussi à créer l’utilisateur admin",
     "ssowat_persistent_conf_read_error": "Impossible de lire la configuration persistante de SSOwat : {error:s}. Modifiez le fichier /etc/ssowat/conf.json.persistent pour réparer la syntaxe JSON",
     "ssowat_persistent_conf_write_error": "Impossible de sauvegarder de la configuration persistante de SSOwat : {error:s}. Modifiez le fichier /etc/ssowat/conf.json.persistent pour réparer la syntaxe JSON",
-    "domain_cannot_remove_main": "Impossible de supprimer le domaine principal. Définissez d'abord un nouveau domaine principal",
+    "domain_cannot_remove_main": "Vous ne pouvez pas supprimer '{domain: s}' car il s'agit du domaine principal. Vous devez d'abord définir un autre domaine comme domaine principal à l'aide de 'yunohost domain main-domain -n <another-domain>', voici la liste des domaines candidats. : {other_domains: s}",
     "certmanager_self_ca_conf_file_not_found": "Le fichier de configuration pour l’autorité du certificat auto-signé est introuvable (fichier : {file:s})",
     "certmanager_unable_to_parse_self_CA_name": "Impossible d’analyser le nom de l’autorité du certificat auto-signé (fichier : {file:s})",
     "mailbox_used_space_dovecot_down": "Le service de courriel Dovecot doit être démarré, si vous souhaitez voir l’espace disque occupé par la messagerie",
@@ -394,7 +394,7 @@
     "migration_0003_system_not_fully_up_to_date": "Votre système n’est pas complètement à jour. Veuillez mener une mise à jour classique avant de lancer à migration à Stretch.",
     "migration_0003_still_on_jessie_after_main_upgrade": "Quelque chose s’est mal passé pendant la mise à niveau principale : le système est toujours sur Debian Jessie !? Pour investiguer sur le problème, veuillez regarder les journaux {log}:s …",
     "migration_0003_general_warning": "Veuillez noter que cette migration est une opération délicate. Si l’équipe YunoHost a fait de son mieux pour la relire et la tester, la migration pourrait tout de même casser des parties de votre système ou de vos applications.\n\nEn conséquence, nous vous recommandons :\n    - de lancer une sauvegarde de vos données ou applications critiques. Plus d’informations sur https://yunohost.org/backup ;\n    - d’être patient après avoir lancé la migration : selon votre connexion internet et matériel, cela pourrait prendre jusqu’à quelques heures pour que tout soit à niveau.\n\nEn outre, le port SMTP utilisé par les clients de messagerie externes comme (Thunderbird ou K9-Mail) a été changé de 465 (SSL/TLS) à 587 (STARTTLS). L’ancien port 465 sera automatiquement fermé et le nouveau port 587 sera ouvert dans le pare-feu. Vous et vos utilisateurs *devront* adapter la configuration de vos clients de messagerie en conséquence.",
-    "migration_0003_problematic_apps_warning": "Veuillez noter que des applications possiblement problématiques ont été détectées. Il semble qu’elles n’aient pas été installées depuis une liste d’application ou qu’elles ne soit pas marquées comme « fonctionnelles ». En conséquence, nous ne pouvons pas garantir qu’elles fonctionneront après la mise à niveau : {problematic_apps}",
+    "migration_0003_problematic_apps_warning": "Veuillez noter que les applications installées potentiellement problématiques suivantes ont été détectées. Il semble que celles-ci n'ont pas été installées à partir d'un catalogue d'applications, ou ne sont pas marquées comme \"working \". Par conséquent, il ne peut pas être garanti qu'ils fonctionneront toujours après la mise à niveau: {problematic_apps}",
     "migration_0003_modified_files": "Veuillez noter que les fichiers suivants ont été détectés comme modifiés manuellement et pourraient être écrasés à la fin de la mise à niveau : {manually_modified_files}",
     "migrations_list_conflict_pending_done": "Vous ne pouvez pas utiliser --previous et --done simultanément.",
     "migrations_to_be_ran_manually": "La migration {id} doit être lancée manuellement. Veuillez aller dans Outils > Migrations dans l’interface admin, ou lancer `yunohost tools migrations migrate`.",
@@ -541,7 +541,7 @@
     "global_settings_setting_security_ssh_compatibility": "Compatibilité versus compromis sécuritaire pour le serveur SSH. Affecte les cryptogrammes (et d'autres aspects liés à la sécurité)",
     "global_settings_setting_security_postfix_compatibility": "Compatibilité versus compromis sécuritaire pour le serveur Postfix. Affecte les cryptogrammes (et d'autres aspects liés à la sécurité)",
     "migration_description_0009_decouple_regenconf_from_services": "Dissocier le mécanisme « regen-conf » des services",
-    "migration_description_0010_migrate_to_apps_json": "Supprimer les listes d'applications obsolètes et utiliser la nouvelle liste unifiée 'apps.json' à la place",
+    "migration_description_0010_migrate_to_apps_json": "Supprimez les catalogues d'applications obsolètes et utilisez à la place la nouvelle liste unifiée 'apps.json' (obsolète, remplacée par la migration 13).",
     "regenconf_file_kept_back": "Le fichier de configuration '{conf}' devait être supprimé par « regen-conf » (catégorie {category}) mais a été conservé.",
     "regenconf_updated": "La configuration a été mise à jour pour la catégorie '{category}'",
     "regenconf_would_be_updated": "La configuration aurait dû être mise à jour pour la catégorie '{category}'",
@@ -625,7 +625,7 @@
     "migrations_running_forward": "Exécution de la migration {id}…",
     "migrations_success_forward": "Migration {id} terminée",
     "need_define_permission_before": "Redéfinissez l'autorisation à l'aide de 'yunohost user permission add -u USER' avant de supprimer un groupe autorisé",
-    "operation_interrupted": "L'opération a-t-elle été interrompue manuellement ?",
+    "operation_interrupted": "L'opération a été interrompue manuellement ?",
     "permission_already_clear": "L'autorisation '{permission: s}' est déjà vide pour l'application {app: s}",
     "permission_already_exist": "L'autorisation '{permission}' existe déjà",
     "permission_created": "Permission '{permission:s}' créée",
@@ -665,5 +665,81 @@
     "app_install_failed": "Impossible d'installer {app}: {error}",
     "app_install_script_failed": "Une erreur est survenue dans le script d'installation de l'application",
     "permission_require_account": "Permission {permission} n'a de sens que pour les utilisateurs ayant un compte et ne peut donc pas être activé pour les visiteurs.",
-    "app_remove_after_failed_install": "Supprimer l'application après l'échec de l'installation…"
+    "app_remove_after_failed_install": "Supprimer l'application après l'échec de l'installation…",
+    "diagnosis_display_tip_web": "Vous pouvez aller à la section Diagnostic (dans l'écran d'accueil) pour voir les problèmes rencontrés.",
+    "diagnosis_cant_run_because_of_dep": "Impossible d'exécuter le diagnostic pour {category} alors qu'il existe des problèmes importants liés à {dep}.",
+    "diagnosis_found_errors": "Trouvé {errors} problème(s) significatif(s) lié(s) à {category} !",
+    "diagnosis_found_errors_and_warnings": "Trouvé {errors} problème(s) significatif(s) (et {warnings} (avertissement(s)) en relation avec {category} !",
+    "diagnosis_ip_not_connected_at_all": "Le serveur ne semble pas du tout connecté à Internet !?",
+    "diagnosis_ip_weird_resolvconf": "La résolution DNS semble fonctionner, mais soyer prudent en utilisant un fichier /etc/resolv.conf personnalisé.",
+    "diagnosis_ip_weird_resolvconf_details": "Au lieu de cela, ce fichier devrait être un lien symbolique vers /etc/resolvconf/run/resolv.conf lui-même pointant vers 127.0.0.1 (dnsmasq). Les résolveurs réels doivent être configurés via /etc/resolv.dnsmasq.conf.",
+    "diagnosis_dns_missing_record": "Selon la configuration DNS recommandée, vous devez ajouter un enregistrement DNS de type {0}, nom {1} et valeur {2}",
+    "diagnosis_diskusage_ok": "Le stockage {mountpoint} (sur le périphérique {device}) a toujours {espace libre {free_abs_GB} GB ({free_percent}%) !",
+    "diagnosis_ram_ok": "Le système dispose toujours de {available_abs_MB} MB ({available_percent}%) de RAM sur {total_abs_MB} MB.",
+    "diagnosis_regenconf_allgood": "Tous les fichiers de configuration sont conformes à la configuration recommandée !",
+    "diagnosis_security_vulnerable_to_meltdown": "Vous semblez vulnérable à la vulnérabilité de sécurité critique de Meltdown",
+    "diagnosis_basesystem_host": "Le serveur exécute Debian {debian_version}.",
+    "diagnosis_basesystem_kernel": "Le serveur exécute le noyau Linux {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} version: {1} ({2})",
+    "diagnosis_basesystem_ynh_main_version": "Le serveur exécute YunoHost {main_version} ({repo})",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "Vous exécutez des versions incohérentes des packages YunoHost ... probablement à cause d'une mise à niveau partielle ou échouée.",
+    "diagnosis_display_tip_cli": "Vous pouvez exécuter 'yunohost diagnosis show --issues' pour afficher les problèmes détectés.",
+    "diagnosis_failed_for_category": "Échec du diagnostic pour la catégorie '{category}' : {error}",
+    "diagnosis_cache_still_valid": "(Le cache est toujours valide pour le diagnostic {category}. Pas re-diagnostiquer pour le moment!)",
+    "diagnosis_ignored_issues": "(+ {nb_ignored} questions ignorée(s))",
+    "diagnosis_found_warnings": "Trouvé {warnings} objet(s) pouvant être amélioré(s) pour {category}.",
+    "diagnosis_everything_ok": "Tout semble bien pour {category} !",
+    "diagnosis_failed": "Impossible d'extraire le résultat du diagnostic pour la catégorie '{category}': {error}",
+    "diagnosis_ip_connected_ipv4": "Le serveur est connecté à Internet via IPv4 !",
+    "diagnosis_ip_no_ipv4": "Le serveur ne dispose pas d’une adresse IPv4 active.",
+    "diagnosis_ip_connected_ipv6": "Le serveur est connecté à Internet via IPv6 !",
+    "diagnosis_ip_no_ipv6": "Le serveur ne dispose pas d'une adresse IPv6 active.",
+    "diagnosis_ip_dnsresolution_working": "La résolution de nom de domaine fonctionne !",
+    "diagnosis_ip_broken_dnsresolution": "La résolution du nom de domaine semble interrompue pour une raison quelconque ... Un pare-feu bloque-t-il les requêtes DNS ?",
+    "diagnosis_ip_broken_resolvconf": "La résolution du nom de domaine semble cassée sur votre serveur, ce qui semble lié au fait que /etc/resolv.conf ne pointe pas vers 127.0.0.1.",
+    "diagnosis_dns_good_conf": "Bonne configuration DNS pour le domaine {domain} (catégorie {category})",
+    "diagnosis_dns_bad_conf": "Configuration DNS incorrecte/manquante pour le domaine {domain} (catégorie {category})",
+    "diagnosis_dns_discrepancy": "Selon la configuration DNS recommandée, la valeur de l'enregistrement DNS de type {0} et nom {1} doit être {2} et non {3}.",
+    "diagnosis_services_bad_status": "Le service {service} est {status} :/",
+    "diagnosis_services_good_status": "Le service {service} est {status} comme prévu !",
+    "diagnosis_diskusage_verylow": "Le stockage {mountpoint} (sur le périphérique {device}) ne dispose que de {free_abs_GB} Go ({free_percent}%). Vous devriez vraiment envisager de nettoyer un peu d'espace.",
+    "diagnosis_diskusage_low": "Le stockage {mountpoint} (sur le périphérique {device}) ne dispose que de {free_abs_GB} Go ({free_percent}%). Faites attention.",
+    "diagnosis_ram_verylow": "Le système ne dispose plus que de {available_abs_MB} MB ({available_percent}%)! (sur {total_abs_MB} Mo)",
+    "diagnosis_ram_low": "Le système n'a plus de {available_abs_MB} MB ({available_percent}%) RAM sur {total_abs_MB} MB. Faites attention.",
+    "diagnosis_swap_none": "Le système n'a aucun échange. Vous devez envisager d’ajouter au moins 256 Mo de swap pour éviter les situations où le système manque de mémoire.",
+    "diagnosis_swap_notsomuch": "Le système ne dispose que de {total_MB} Mo de swap. Vous devez envisager d'avoir au moins 256 Mo pour éviter les situations où le système manque de mémoire.",
+    "diagnosis_swap_ok": "Le système dispose de {total_MB} Mo de swap !",
+    "diagnosis_regenconf_manually_modified": "Le fichier de configuration {file} a été modifié manuellement.",
+    "diagnosis_regenconf_manually_modified_debian": "Le fichier de configuration {file} a été modifié manuellement par rapport à celui par défaut de Debian.",
+    "diagnosis_regenconf_manually_modified_details": "C'est probablement OK tant que vous savez ce que vous faites;) !",
+    "diagnosis_regenconf_manually_modified_debian_details": "Cela peut probablement être OK, mais il faut garder un œil dessus ...",
+    "diagnosis_regenconf_nginx_conf_broken": "La configuration de nginx semble être cassée !",
+    "diagnosis_security_all_good": "Aucune vulnérabilité de sécurité critique n'a été trouvée.",
+    "apps_catalog_init_success": "Système de catalogue d'applications initialisé !",
+    "apps_catalog_failed_to_download": "Impossible de télécharger le catalogue des applications {apps_catalog}:{error}",
+    "diagnosis_mail_ougoing_port_25_blocked": "Le port sortant 25 semble être bloqué. Vous devriez essayer de le débloquer dans le panneau de configuration de votre fournisseur de services Internet (ou hébergeur). En attendant, le serveur ne pourra pas envoyer de courrier électronique à d'autres serveurs.",
+    "domain_cannot_remove_main_add_new_one": "Vous ne pouvez pas supprimer '{domain:s}' car il s'agit du domaine principal et de votre seul domaine. Vous devez d'abord ajouter un autre domaine à l'aide de 'yunohost domain add <another-domain.com>', puis définir comme domaine principal à l'aide de ' yunohost domain main-domain -n <nomd'un-autre-domaine.com>' et vous pouvez ensuite supprimer le domaine '{domaine: s}' à l'aide de 'yunohost domain remove {domain:s}'.'",
+    "diagnosis_security_vulnerable_to_meltdown_details": "Pour résoudre ce problème, vous devez mettre à niveau votre système et redémarrer pour charger le nouveau noyau Linux (ou contacter votre fournisseur de serveur si cela ne fonctionne pas). Voir https://meltdownattack.com/ pour plus d'informations.",
+    "diagnosis_description_basesystem": "Système de base",
+    "diagnosis_description_ip": "Connectivité Internet",
+    "diagnosis_description_dnsrecords": "Enregistrements DNS",
+    "diagnosis_description_services": "Vérification de l'état des services",
+    "diagnosis_description_systemresources": "Ressources système",
+    "diagnosis_description_ports": "Exposition des ports",
+    "diagnosis_description_http": "Exposition HTTP",
+    "diagnosis_description_regenconf": "Configurations système",
+    "diagnosis_description_security": "Contrôles de sécurité",
+    "diagnosis_ports_could_not_diagnose": "Impossible de diagnostiquer si les ports sont accessibles de l'extérieur. Erreur: {error}",
+    "apps_catalog_updating": "Mise à jour du catalogue d'applications...",
+    "apps_catalog_obsolete_cache": "Le cache du catalogue d'applications est vide ou obsolète.",
+    "apps_catalog_update_success": "Le catalogue des applications a été mis à jour !",
+    "diagnosis_mail_ougoing_port_25_ok": "Le port sortant 25 n'est pas bloqué et le courrier électronique peut être envoyé à d'autres serveurs.",
+    "diagnosis_description_mail": "Email",
+    "diagnosis_ports_unreachable": "Le port {port} n'est pas accessible de l'extérieur.",
+    "diagnosis_ports_ok": "Le port {port} est accessible de l'extérieur.",
+    "diagnosis_http_could_not_diagnose": "Impossible de diagnostiquer si le domaine est accessible de l'extérieur. Erreur: {error}",
+    "diagnosis_http_ok": "Le domaine {domain} est accessible de l'extérieur.",
+    "diagnosis_http_unreachable": "Le domaine {domain} est inaccessible via HTTP de l'extérieur.",
+    "diagnosis_unknown_categories": "Les catégories suivantes sont inconnues: {categories}",
+    "migration_description_0013_futureproof_apps_catalog_system": "Migrer vers le nouveau système de catalogue d'applications à l'épreuve du temps"
 }

locales/oc.json

@@ -5,7 +5,7 @@
     "app_already_installed": "{app:s} es ja installat",
     "app_already_up_to_date": "{app:s} es ja a jorn",
     "installation_complete": "Installacion acabada",
-    "app_id_invalid": "Id d’aplicacion incorrècte",
+    "app_id_invalid": "ID d’aplicacion incorrècte",
     "app_install_files_invalid": "Fichièrs d’installacion incorrèctes",
     "app_no_upgrade": "Pas cap d’aplicacion d’actualizar",
     "app_not_correctly_installed": "{app:s} sembla pas ben installat",
@@ -41,15 +41,15 @@
     "backup_archive_name_unknown": "L’archiu local de salvagarda apelat  « {name:s} » es desconegut",
     "action_invalid": "Accion « {action:s} » incorrècta",
     "app_argument_choice_invalid": "Causida invalida pel paramètre  « {name:s} », cal que siá un de {choices:s}",
-    "app_argument_invalid": "Valor invalida pel paramètre  « {name:s} » : {error:s}",
+    "app_argument_invalid": "Causissètz una valor invalida pel paramètre  « {name:s} » : {error:s}",
     "app_argument_required": "Lo paramètre « {name:s} » es requesit",
-    "app_change_url_failed_nginx_reload": "La reaviada de nginx a fracassat. Vaquí la sortida de « nginx -t » :\n{nginx_errors:s}",
+    "app_change_url_failed_nginx_reload": "Reaviada de NGINX impossibla. Vaquí la sortida de « nginx -t » :\n{nginx_errors:s}",
     "app_change_url_identical_domains": "L’ancian e lo novèl coble domeni/camin son identics per {domain:s}{path:s}, pas res a far.",
-    "app_change_url_success": "L’URL de l’aplicacion {app:s} a cambiat per {domain:s}{path:s}",
+    "app_change_url_success": "L’URL de l’aplicacion {app:s} es ara {domain:s}{path:s}",
     "app_checkurl_is_deprecated": "Packagers /!\\  ’app checkurl’ es obsolèt ! Utilizatz ’app register-url’ a la plaça !",
     "app_extraction_failed": "Extraccion dels fichièrs d’installacion impossibla",
     "app_incompatible": "L’aplicacion {app} es pas compatibla amb vòstra version de YunoHost",
-    "app_location_already_used": "L’aplicacion « {app} » es ja installada a aqueste emplaçament ({path})",
+    "app_location_already_used": "L’aplicacion « {app} » es ja installada dins ({path})",
     "app_manifest_invalid": "Manifest d’aplicacion incorrècte : {error}",
     "app_package_need_update": "Lo paquet de l’aplicacion {app} deu èsser actualizat per poder seguir los cambiaments de YunoHost",
     "app_requirements_checking": "Verificacion dels paquets requesits per {app}…",
@@ -612,5 +612,77 @@
     "migrations_must_provide_explicit_targets": "Devètz fornir una cibla explicita quand utilizatz  using --skip o --force-rerun",
     "migrations_exclusive_options": "--auto, --skip, e --force-rerun son las opcions exclusivas.",
     "migrations_failed_to_load_migration": "Cargament impossible de la migracion {id} : {error}",
-    "migrations_already_ran": "Aquelas migracions s’executèron ja : {ids}"
+    "migrations_already_ran": "Aquelas migracions s’executèron ja : {ids}",
+    "diagnosis_basesystem_ynh_main_version": "Lo servidor fonciona amb YunoHost {main_version} ({repo})",
+    "migrations_dependencies_not_satisfied": "Executatz aquestas migracions : « {dependencies_id} », abans la migracion {id}.",
+    "migrations_no_such_migration": "I a pas cap de migracion apelada « {id} »",
+    "migrations_not_pending_cant_skip": "Aquestas migracions son pas en espèra, las podètz pas doncas ignorar : {ids}",
+    "app_action_broke_system": "Aquesta accion sembla aver copat de servicis importants : {services}",
+    "diagnosis_display_tip_web": "Podètz anar a la seccion Diagnostic (dins l’ecran d’acuèlh) per veire los problèmas trobats.",
+    "diagnosis_ip_no_ipv6": "Lo servidor a pas d’adreça IPv5 activa.",
+    "diagnosis_ip_not_connected_at_all": "Lo servidor sembla pas connectat a Internet ?!",
+    "diagnosis_security_all_good": "Cap de vulnerabilitat de seguretat critica pas trobada.",
+    "diagnosis_description_regenconf": "Configuracion sistèma",
+    "diagnosis_http_ok": "Lo domeni {domain} accessible de l’exterior.",
+    "app_full_domain_unavailable": "Aquesta aplicacion a d’èsser installada sul seu pròpri domeni, mas i a d’autras aplicacions installadas sus aqueste domeni « {domain} ». Podètz utilizar allòc un josdomeni dedicat a aquesta aplicacion.",
+    "app_upgrade_stopped": "L’actualizacion de totas las aplicacions s‘es arrestada per evitar de possibles damatges pr’amor qu’èra pas possible d’actualizar una aplicacion",
+    "diagnosis_dns_bad_conf": "Configuracion DNS incorrècta o inexistenta pel domeni {domain} (categoria {category})",
+    "diagnosis_ram_verylow": "Lo sistèma a solament {available_abs_MB} Mo ({available_percent}%) de memòria RAM disponibla ! (d’un total de {total_abs_MB} MB)",
+    "diagnosis_ram_ok": "Lo sistèma a encara {available_abs_MB} Mo ({available_percent}%) de memòria RAM disponibla d’un total de {total_abs_MB} MB).",
+    "permission_already_allowed": "Lo grop « {group} » a ja la permission « {permission} » activada",
+    "permission_already_disallowed": "Lo grop « {group} » a ja la permission « {permission} » desactivada",
+    "permission_cannot_remove_main": "La supression d’una permission màger es pas autorizada",
+    "log_permission_url": "Actualizacion de l’URL ligada a la permission « {} »",
+    "app_install_failed": "Installacion impossibla de {app} : {error}",
+    "app_install_script_failed": "Una error s’es producha en installar lo script de l’aplicacion",
+    "migration_0011_failed_to_remove_stale_object": "Supression impossibla d’un objècte obsolèt {dn} : {error}",
+    "apps_already_up_to_date": "Totas las aplicacions son ja al jorn",
+    "app_remove_after_failed_install": "Supression de l’aplicacion aprèp fracàs de l’installacion…",
+    "group_already_exist": "Lo grop {group} existís ja",
+    "group_already_exist_on_system": "Lo grop {group} existís ja dins lo sistèma de grops",
+    "group_user_not_in_group": "L’utilizaire {user} es pas dins lo grop {group}",
+    "log_user_permission_reset": "Restablir la permission « {} »",
+    "user_already_exists": "L’utilizaire {user} existís ja",
+    "diagnosis_basesystem_host": "Lo servidor fonciona amb Debian {debian_version}.",
+    "diagnosis_basesystem_kernel": "Lo servidor fonciona amb lo nuclèu Linuxl {kernel_version}",
+    "diagnosis_basesystem_ynh_single_version": "{0} version : {1} ({2})",
+    "diagnosis_basesystem_ynh_inconsistent_versions": "Utilizatz de versions inconsistentas dels paquets de YunoHost… probablament a causa d'una actualizacion fracassada o parciala.",
+    "diagnosis_display_tip_cli": "Podètz executar « yunohost diagnosis show --issues » per mostrar las errors trobadas.",
+    "diagnosis_ignored_issues": "(+ {nb_ignored} problèma(es) ignorat(s))",
+    "diagnosis_everything_ok": "Tot sembla corrècte per {category} !",
+    "diagnosis_ip_connected_ipv4": "Lo servidor es connectat a Internet via IPv4 !",
+    "diagnosis_ip_no_ipv4": "Lo servidor a pas d’adreça IPv4 activa.",
+    "diagnosis_ip_connected_ipv6": "Lo servidor es connectat a Internet via IPv6 !",
+    "diagnosis_ip_dnsresolution_working": "La resolucion del nom de domeni fonciona !",
+    "diagnosis_dns_good_conf": "Bona configuracion DNS pel domeni {domain} (categoria {category})",
+    "diagnosis_failed_for_category": "Lo diagnostic a reüssit per la categoria « {category} » : {error}",
+    "diagnosis_cache_still_valid": "(Memòria cache totjorn valida pel diagnostic {category}. Cap d’autre diagnostic pel moment !)",
+    "diagnosis_found_errors": "{errors} errors importantas trobadas ligadas a {category} !",
+    "diagnosis_services_good_status": "Lo servici {service} es {status} coma previst !",
+    "diagnosis_services_bad_status": "Lo servici {service} es {status} :/",
+    "diagnosis_swap_ok": "Lo sistèma a {total_MB} MB d’escambi !",
+    "diagnosis_regenconf_allgood": "Totes los fichièrs de configuracion son confòrmes a la configuracion recomandada !",
+    "diagnosis_regenconf_manually_modified": "Lo fichièr de configuracion {file} foguèt modificat manualament.",
+    "diagnosis_regenconf_manually_modified_details": "Es probablament bon tan que sabètz çò que fasètz ;) !",
+    "diagnosis_regenconf_nginx_conf_broken": "La configuracion de nginx sembla èsser copada !",
+    "diagnosis_security_vulnerable_to_meltdown": "Semblatz èsser vulnerable a la vulnerabilitat de seguretat critica de Meltdown",
+    "diagnosis_description_basesystem": "Sistèma de basa",
+    "diagnosis_description_ip": "Connectivitat Internet",
+    "diagnosis_description_dnsrecords": "Enregistraments DNS",
+    "diagnosis_description_services": "Verificacion d’estat de servicis",
+    "diagnosis_description_systemresources": "Resorgas sistèma",
+    "diagnosis_description_ports": "Exposicion dels pòrts",
+    "diagnosis_description_http": "Exposicion HTTP",
+    "diagnosis_description_security": "Verificacion de seguretat",
+    "diagnosis_ports_unreachable": "Lo pòrt {port} es pas accessible de l’exterior.",
+    "diagnosis_ports_ok": "Lo pòrt {port} es accessible de l’exterior.",
+    "diagnosis_http_unreachable": "Lo domeni {domain} es pas accessible via HTTP de l’exterior.",
+    "diagnosis_unknown_categories": "La categorias seguentas son desconegudas : {categories}",
+    "diagnosis_ram_low": "Lo sistèma a {available_abs_MB} Mo ({available_percent}%) de memòria RAM disponibla d’un total de {total_abs_MB} MB). Atencion.",
+    "diagnosis_regenconf_manually_modified_debian": "Lo fichier de configuracion {file} foguèt modificat manualament respècte al fichièr per defaut de Debian.",
+    "log_permission_create": "Crear la permission « {} »",
+    "log_permission_delete": "Suprimir la permission « {} »",
+    "log_user_group_create": "Crear lo grop « {} »",
+    "log_user_permission_update": "Actualizacion dels accèsses per la permission « {} »",
+    "operation_interrupted": "L’operacion es estada interrompuda manualament ?"
 }

src/yunohost/app.py

@@ -33,15 +33,14 @@ import re
 import urlparse
 import subprocess
 import glob
-import pwd
-import grp
 import urllib
 from collections import OrderedDict
 from datetime import datetime
 
 from moulinette import msignals, m18n, msettings
 from moulinette.utils.log import getActionLogger
-from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_json
+from moulinette.utils.network import download_json
+from moulinette.utils.filesystem import read_json, read_toml, read_yaml, write_to_file, write_to_json, write_to_yaml, chmod, chown, mkdir
 
 from yunohost.service import service_log, service_status, _run_service_command
 from yunohost.utils import packages
@@ -50,12 +49,16 @@ from yunohost.log import is_unit_operation, OperationLogger
 
 logger = getActionLogger('yunohost.app')
 
-REPO_PATH = '/var/cache/yunohost/repo'
 APPS_PATH = '/usr/share/yunohost/apps'
 APPS_SETTING_PATH = '/etc/yunohost/apps/'
 INSTALL_TMP = '/var/cache/yunohost'
 APP_TMP_FOLDER = INSTALL_TMP + '/from_file'
-APPSLISTS_JSON = '/etc/yunohost/appslists.json'
+
+APPS_CATALOG_CACHE = '/var/cache/yunohost/repo'
+APPS_CATALOG_CONF = '/etc/yunohost/apps_catalog.yml'
+APPS_CATALOG_CRON_PATH = "/etc/cron.daily/yunohost-fetch-apps-catalog"
+APPS_CATALOG_API_VERSION = 1
+APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default"
 
 re_github_repo = re.compile(
     r'^(http[s]?://|git@)github.com[/:]'
@@ -68,166 +71,6 @@ re_app_instance_name = re.compile(
 )
 
 
-def app_listlists():
-    """
-    List fetched lists
-
-    """
-
-    # Migrate appslist system if needed
-    # XXX move to a migration when those are implemented
-    if _using_legacy_appslist_system():
-        _migrate_appslist_system()
-
-    # Get the list
-    appslist_list = _read_appslist_list()
-
-    # Convert 'lastUpdate' timestamp to datetime
-    for name, infos in appslist_list.items():
-        if infos["lastUpdate"] is None:
-            infos["lastUpdate"] = 0
-        infos["lastUpdate"] = datetime.utcfromtimestamp(infos["lastUpdate"])
-
-    return appslist_list
-
-
-def app_fetchlist(url=None, name=None):
-    """
-    Fetch application list(s) from app server. By default, fetch all lists.
-
-    Keyword argument:
-        name -- Name of the list
-        url -- URL of remote JSON list
-    """
-    if url and not url.endswith(".json"):
-        raise YunohostError("This is not a valid application list url. It should end with .json.")
-
-    # If needed, create folder where actual appslists are stored
-    if not os.path.exists(REPO_PATH):
-        os.makedirs(REPO_PATH)
-
-    # Migrate appslist system if needed
-    # XXX move that to a migration once they are finished
-    if _using_legacy_appslist_system():
-        _migrate_appslist_system()
-
-    # Read the list of appslist...
-    appslists = _read_appslist_list()
-
-    # Determine the list of appslist to be fetched
-    appslists_to_be_fetched = []
-
-    # If a url and and a name is given, try to register new list,
-    # the fetch only this list
-    if url is not None:
-        if name:
-            operation_logger = OperationLogger('app_fetchlist')
-            operation_logger.start()
-            _register_new_appslist(url, name)
-            # Refresh the appslists dict
-            appslists = _read_appslist_list()
-            appslists_to_be_fetched = [name]
-            operation_logger.success()
-        else:
-            raise YunohostError('custom_appslist_name_required')
-
-    # If a name is given, look for an appslist with that name and fetch it
-    elif name is not None:
-        if name not in appslists.keys():
-            raise YunohostError('appslist_unknown', appslist=name)
-        else:
-            appslists_to_be_fetched = [name]
-
-    # Otherwise, fetch all lists
-    else:
-        appslists_to_be_fetched = appslists.keys()
-
-    import requests  # lazy loading this module for performance reasons
-    # Fetch all appslists to be fetched
-    for name in appslists_to_be_fetched:
-
-        url = appslists[name]["url"]
-
-        logger.debug("Attempting to fetch list %s at %s" % (name, url))
-
-        # Download file
-        try:
-            appslist_request = requests.get(url, timeout=30)
-        except requests.exceptions.SSLError:
-            logger.error(m18n.n('appslist_retrieve_error',
-                                appslist=name,
-                                error="SSL connection error"))
-            continue
-        except Exception as e:
-            logger.error(m18n.n('appslist_retrieve_error',
-                                appslist=name,
-                                error=str(e)))
-            continue
-        if appslist_request.status_code != 200:
-            logger.error(m18n.n('appslist_retrieve_error',
-                                appslist=name,
-                                error="Server returned code %s " %
-                                str(appslist_request.status_code)))
-            continue
-
-        # Validate app list format
-        # TODO / Possible improvement : better validation for app list (check
-        # that json fields actually look like an app list and not any json
-        # file)
-        appslist = appslist_request.text
-        try:
-            json.loads(appslist)
-        except ValueError as e:
-            logger.error(m18n.n('appslist_retrieve_bad_format',
-                                appslist=name))
-            continue
-
-        # Write app list to file
-        list_file = '%s/%s.json' % (REPO_PATH, name)
-        try:
-            with open(list_file, "w") as f:
-                f.write(appslist)
-        except Exception as e:
-            raise YunohostError("Error while writing appslist %s: %s" % (name, str(e)), raw_msg=True)
-
-        now = int(time.time())
-        appslists[name]["lastUpdate"] = now
-
-        logger.success(m18n.n('appslist_fetched', appslist=name))
-
-    # Write updated list of appslist
-    _write_appslist_list(appslists)
-
-
-@is_unit_operation()
-def app_removelist(operation_logger, name):
-    """
-    Remove list from the repositories
-
-    Keyword argument:
-        name -- Name of the list to remove
-
-    """
-    appslists = _read_appslist_list()
-
-    # Make sure we know this appslist
-    if name not in appslists.keys():
-        raise YunohostError('appslist_unknown', appslist=name)
-
-    operation_logger.start()
-
-    # Remove json
-    json_path = '%s/%s.json' % (REPO_PATH, name)
-    if os.path.exists(json_path):
-        os.remove(json_path)
-
-    # Forget about this appslist
-    del appslists[name]
-    _write_appslist_list(appslists)
-
-    logger.success(m18n.n('appslist_removed', appslist=name))
-
-
 def app_list(filter=None, raw=False, installed=False, with_backup=False):
     """
     List apps
@@ -243,28 +86,10 @@ def app_list(filter=None, raw=False, installed=False, with_backup=False):
     """
     installed = with_backup or installed
 
-    app_dict = {}
     list_dict = {} if raw else []
 
-    appslists = _read_appslist_list()
+    # Get app list from catalog cache
-
+    app_dict = _load_apps_catalog()
-    for appslist in appslists.keys():
-
-        json_path = "%s/%s.json" % (REPO_PATH, appslist)
-
-        # If we don't have the json yet, try to fetch it
-        if not os.path.exists(json_path):
-            app_fetchlist(name=appslist)
-
-        # If it now exist
-        if os.path.exists(json_path):
-            appslist_content = read_json(json_path)
-            for app, info in appslist_content.items():
-                if app not in app_dict:
-                    info['repository'] = appslist
-                    app_dict[app] = info
-        else:
-            logger.warning("Uh there's no data for applist '%s' ... (That should be just a temporary issue?)" % appslist)
 
     # Get app list from the app settings directory
     for app in os.listdir(APPS_SETTING_PATH):
@@ -405,6 +230,7 @@ def app_map(app=None, raw=False, user=None):
         app -- Specific app to map
 
     """
+
     from yunohost.permission import user_permission_list
 
     apps = []
@@ -559,7 +385,7 @@ def app_change_url(operation_logger, app, domain, path):
     # Retrieve arguments list for change_url script
     # TODO: Allow to specify arguments
     args_odict = _parse_args_from_manifest(manifest, 'change_url')
-    args_list = [ value[0] for value in args_odict.values() ]
+    args_list = [value[0] for value in args_odict.values()]
     args_list.append(app)
 
     # Prepare env. var. to pass to script
@@ -612,7 +438,7 @@ def app_change_url(operation_logger, app, domain, path):
     app_setting(app, 'domain', value=domain)
     app_setting(app, 'path', value=path)
 
-    app_ssowatconf()
+    permission_update(app, permission="main", add_url=[domain + path], remove_url=[old_domain + old_path], sync_perm=True)
 
     # avoid common mistakes
     if _run_service_command("reload", "nginx") is False:
@@ -656,15 +482,15 @@ def app_upgrade(app=[], url=None, file=None):
     if not apps:
         # FIXME : not sure what's supposed to happen if there is a url and a file but no apps...
         if not url and not file:
-            apps = [app["id"] for app in app_list(installed=True)["apps"]]
+            apps = [app_["id"] for app_ in app_list(installed=True)["apps"]]
     elif not isinstance(app, list):
         apps = [app]
 
     # Remove possible duplicates
-    apps = [app for i,app in enumerate(apps) if apps not in apps[:i]]
+    apps = [app_ for i, app_ in enumerate(apps) if app_ not in apps[:i]]
 
     # Abort if any of those app is in fact not installed..
-    for app in [app for app in apps if not _is_installed(app)]:
+    for app in [app_ for app_ in apps if not _is_installed(app_)]:
         raise YunohostError('app_not_installed', app=app, all_apps=_get_all_installed_apps_id())
 
     if len(apps) == 0:
@@ -706,7 +532,7 @@ def app_upgrade(app=[], url=None, file=None):
         # Retrieve arguments list for upgrade script
         # TODO: Allow to specify arguments
         args_odict = _parse_args_from_manifest(manifest, 'upgrade')
-        args_list = [ value[0] for value in args_odict.values() ]
+        args_list = [value[0] for value in args_odict.values()]
         args_list.append(app_instance_name)
 
         # Prepare env. var. to pass to script
@@ -858,8 +684,6 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
             if answer.upper() != "Y":
                 raise YunohostError("aborting")
 
-
-
     raw_app_list = app_list(raw=True)
 
     if app in raw_app_list or ('@' in app) or ('http://' in app) or ('https://' in app):
@@ -920,7 +744,7 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
     args_dict = {} if not args else \
         dict(urlparse.parse_qsl(args, keep_blank_values=True))
     args_odict = _parse_args_from_manifest(manifest, 'install', args=args_dict)
-    args_list = [ value[0] for value in args_odict.values() ]
+    args_list = [value[0] for value in args_odict.values()]
     args_list.append(app_instance_name)
 
     # Validate domain / path availability for webapps
@@ -938,8 +762,8 @@ def app_install(operation_logger, app, label=None, args=None, no_remove_on_failu
     # Tell the operation_logger to redact all password-type args
     # Also redact the % escaped version of the password that might appear in
     # the 'args' section of metadata (relevant for password with non-alphanumeric char)
-    data_to_redact = [ value[0] for value in args_odict.values() if value[1] == "password" ]
+    data_to_redact = [value[0] for value in args_odict.values() if value[1] == "password"]
-    data_to_redact += [ urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data ]
+    data_to_redact += [urllib.quote(data) for data in data_to_redact if urllib.quote(data) != data]
     operation_logger.data_to_redact.extend(data_to_redact)
 
     operation_logger.related_to = [s for s in operation_logger.related_to if s[0] != "app"]
@@ -1165,7 +989,7 @@ def app_remove(operation_logger, app):
     # TODO: display fail messages from script
     try:
         shutil.rmtree('/tmp/yunohost_remove')
-    except:
+    except Exception:
         pass
 
     # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove
@@ -2073,8 +1897,7 @@ def _get_app_config_panel(app_id):
             "panel": [],
         }
 
-        panels = filter(lambda (key, value): key not in ("name", "version")
+        panels = filter(lambda (key, value): key not in ("name", "version") and isinstance(value, OrderedDict),
-                                             and isinstance(value, OrderedDict),
                         toml_config_panel.items())
 
         for key, value in panels:
@@ -2084,8 +1907,7 @@ def _get_app_config_panel(app_id):
                 "sections": [],
             }
 
-            sections = filter(lambda (k, v): k not in ("name",)
+            sections = filter(lambda (k, v): k not in ("name",) and isinstance(v, OrderedDict),
-                                             and isinstance(v, OrderedDict),
                               value.items())
 
             for section_key, section_value in sections:
@@ -2095,8 +1917,7 @@ def _get_app_config_panel(app_id):
                     "options": [],
                 }
 
-                options = filter(lambda (k, v): k not in ("name",)
+                options = filter(lambda (k, v): k not in ("name",) and isinstance(v, OrderedDict),
-                                                and isinstance(v, OrderedDict),
                                  section_value.items())
 
                 for option_key, option_value in options:
@@ -2912,151 +2733,160 @@ def _parse_app_instance_name(app_instance_name):
     return (appid, app_instance_nb)
 
 
-def _using_legacy_appslist_system():
+#
+# ############################### #
+#  Applications list management   #
+# ############################### #
+#
+
+
+def _initialize_apps_catalog_system():
     """
-    Return True if we're using the old fetchlist scheme.
+    This function is meant to intialize the apps_catalog system with YunoHost's default app catalog.
-    This is determined by the presence of some cron job yunohost-applist-foo
+
+    It also creates the cron job that will update the list every day
     """
 
-    return glob.glob("/etc/cron.d/yunohost-applist-*") != []
+    default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}]
-
-
-def _migrate_appslist_system():
-    """
-    Migrate from the legacy fetchlist system to the new one
-    """
-    legacy_crons = glob.glob("/etc/cron.d/yunohost-applist-*")
-
-    for cron_path in legacy_crons:
-        appslist_name = os.path.basename(cron_path).replace("yunohost-applist-", "")
-        logger.debug(m18n.n('appslist_migrating', appslist=appslist_name))
-
-        # Parse appslist url in cron
-        cron_file_content = open(cron_path).read().strip()
-        appslist_url_parse = re.search("-u (https?://[^ ]+)", cron_file_content)
-
-        # Abort if we did not find an url
-        if not appslist_url_parse or not appslist_url_parse.groups():
-            # Bkp the old cron job somewhere else
-            bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
-            os.rename(cron_path, bkp_file)
-            # Notice the user
-            logger.warning(m18n.n('appslist_could_not_migrate',
-                           appslist=appslist_name,
-                           bkp_file=bkp_file))
-        # Otherwise, register the list and remove the legacy cron
-        else:
-            appslist_url = appslist_url_parse.groups()[0]
-            try:
-                _register_new_appslist(appslist_url, appslist_name)
-            # Might get an exception if two legacy cron jobs conflict
-            # in terms of url...
-            except Exception as e:
-                logger.error(str(e))
-                # Bkp the old cron job somewhere else
-                bkp_file = "/etc/yunohost/%s.oldlist.bkp" % appslist_name
-                os.rename(cron_path, bkp_file)
-                # Notice the user
-                logger.warning(m18n.n('appslist_could_not_migrate',
-                               appslist=appslist_name,
-                               bkp_file=bkp_file))
-            else:
-                os.remove(cron_path)
-
-
-def _install_appslist_fetch_cron():
-
-    cron_job_file = "/etc/cron.daily/yunohost-fetch-appslists"
-
-    logger.debug("Installing appslist fetch cron job")
 
     cron_job = []
     cron_job.append("#!/bin/bash")
     # We add a random delay between 0 and 60 min to avoid every instance fetching
-    # the appslist at the same time every night
+    # the apps catalog at the same time every night
     cron_job.append("(sleep $((RANDOM%3600));")
-    cron_job.append("yunohost app fetchlist > /dev/null 2>&1) &")
+    cron_job.append("yunohost tools update --apps > /dev/null) &")
-
-    with open(cron_job_file, "w") as f:
-        f.write('\n'.join(cron_job))
-
-    _set_permissions(cron_job_file, "root", "root", 0o755)
-
-
-# FIXME - Duplicate from certificate.py, should be moved into a common helper
-# thing...
-def _set_permissions(path, user, group, permissions):
-    uid = pwd.getpwnam(user).pw_uid
-    gid = grp.getgrnam(group).gr_gid
-
-    os.chown(path, uid, gid)
-    os.chmod(path, permissions)
-
-
-def _read_appslist_list():
-    """
-    Read the json corresponding to the list of appslists
-    """
-
-    # If file does not exists yet, return empty dict
-    if not os.path.exists(APPSLISTS_JSON):
-        return {}
-
-    # Read file content
-    with open(APPSLISTS_JSON, "r") as f:
-        appslists_json = f.read()
-
-    # Parse json, throw exception if what we got from file is not a valid json
     try:
-        appslists = json.loads(appslists_json)
+        logger.debug("Initializing apps catalog system with YunoHost's default app list")
-    except ValueError:
+        write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list)
-        raise YunohostError('appslist_corrupted_json', filename=APPSLISTS_JSON)
 
-    return appslists
+        logger.debug("Installing apps catalog fetch daily cron job")
-
+        write_to_file(APPS_CATALOG_CRON_PATH, '\n'.join(cron_job))
-
+        chown(APPS_CATALOG_CRON_PATH, uid="root", gid="root")
-def _write_appslist_list(appslist_lists):
+        chmod(APPS_CATALOG_CRON_PATH, 0o755)
-    """
-    Update the json containing list of appslists
-    """
-
-    # Write appslist list
-    try:
-        with open(APPSLISTS_JSON, "w") as f:
-            json.dump(appslist_lists, f)
     except Exception as e:
-        raise YunohostError("Error while writing list of appslist %s: %s" %
+        raise YunohostError("Could not initialize the apps catalog system... : %s" % str(e))
-                            (APPSLISTS_JSON, str(e)), raw_msg=True)
+
+    logger.success(m18n.n("apps_catalog_init_success"))
 
 
-def _register_new_appslist(url, name):
+def _read_apps_catalog_list():
     """
-    Add a new appslist to be fetched regularly.
+    Read the json corresponding to the list of apps catalogs
-    Raise an exception if url or name conflicts with an existing list.
     """
 
-    appslist_list = _read_appslist_list()
+    # Legacy code - can be removed after moving to buster (if the migration got merged before buster)
+    if os.path.exists('/etc/yunohost/appslists.json'):
+        from yunohost.tools import _get_migration_by_name
+        migration = _get_migration_by_name("futureproof_apps_catalog_system")
+        migration.migrate()
 
-    # Check if name conflicts with an existing list
+    try:
-    if name in appslist_list:
+        list_ = read_yaml(APPS_CATALOG_CONF)
-        raise YunohostError('appslist_name_already_tracked', name=name)
+        # Support the case where file exists but is empty
+        # by returning [] if list_ is None
+        return list_ if list_ else []
+    except Exception as e:
+        raise YunohostError("Could not read the apps_catalog list ... : %s" % str(e))
 
-    # Check if url conflicts with an existing list
-    known_appslist_urls = [appslist["url"] for _, appslist in appslist_list.items()]
 
-    if url in known_appslist_urls:
+def _actual_apps_catalog_api_url(base_url):
-        raise YunohostError('appslist_url_already_tracked', url=url)
 
-    logger.debug("Registering new appslist %s at %s" % (name, url))
+    return "{base_url}/v{version}/apps.json".format(base_url=base_url, version=APPS_CATALOG_API_VERSION)
 
-    appslist_list[name] = {
-        "url": url,
-        "lastUpdate": None
-    }
 
-    _write_appslist_list(appslist_list)
+def _update_apps_catalog():
+    """
+    Fetches the json for each apps_catalog and update the cache
 
-    _install_appslist_fetch_cron()
+    apps_catalog_list is for example :
+     [   {"id": "default", "url": "https://app.yunohost.org/default/"}  ]
+
+    Then for each apps_catalog, the actual json URL to be fetched is like :
+       https://app.yunohost.org/default/vX/apps.json
+
+    And store it in :
+        /var/cache/yunohost/repo/default.json
+    """
+
+    apps_catalog_list = _read_apps_catalog_list()
+
+    logger.info(m18n.n("apps_catalog_updating"))
+
+    # Create cache folder if needed
+    if not os.path.exists(APPS_CATALOG_CACHE):
+        logger.debug("Initialize folder for apps catalog cache")
+        mkdir(APPS_CATALOG_CACHE, mode=0o750, parents=True, uid='root')
+
+    for apps_catalog in apps_catalog_list:
+        apps_catalog_id = apps_catalog["id"]
+        actual_api_url = _actual_apps_catalog_api_url(apps_catalog["url"])
+
+        # Fetch the json
+        try:
+            apps_catalog_content = download_json(actual_api_url)
+        except Exception as e:
+            raise YunohostError("apps_catalog_failed_to_download", apps_catalog=apps_catalog_id, error=str(e))
+
+        # Remember the apps_catalog api version for later
+        apps_catalog_content["from_api_version"] = APPS_CATALOG_API_VERSION
+
+        # Save the apps_catalog data in the cache
+        cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id)
+        try:
+            write_to_json(cache_file, apps_catalog_content)
+        except Exception as e:
+            raise YunohostError("Unable to write cache data for %s apps_catalog : %s" % (apps_catalog_id, str(e)))
+
+    logger.success(m18n.n("apps_catalog_update_success"))
+
+
+def _load_apps_catalog():
+    """
+    Read all the apps catalog cache files and build a single dict (app_dict)
+    corresponding to all known apps in all indexes
+    """
+
+    app_dict = {}
+
+    for apps_catalog_id in [L["id"] for L in _read_apps_catalog_list()]:
+
+        # Let's load the json from cache for this catalog
+        cache_file = "{cache_folder}/{list}.json".format(cache_folder=APPS_CATALOG_CACHE, list=apps_catalog_id)
+
+        try:
+            apps_catalog_content = read_json(cache_file) if os.path.exists(cache_file) else None
+        except Exception as e:
+            raise ("Unable to read cache for apps_catalog %s : %s" % (apps_catalog_id, str(e)))
+
+        # Check that the version of the data matches version ....
+        # ... otherwise it means we updated yunohost in the meantime
+        # and need to update the cache for everything to be consistent
+        if not apps_catalog_content or apps_catalog_content.get("from_api_version") != APPS_CATALOG_API_VERSION:
+            logger.info(m18n.n("apps_catalog_obsolete_cache"))
+            _update_apps_catalog()
+            apps_catalog_content = read_json(cache_file)
+
+        del apps_catalog_content["from_api_version"]
+
+        # Add apps from this catalog to the output
+        for app, info in apps_catalog_content.items():
+
+            # (N.B. : there's a small edge case where multiple apps catalog could be listing the same apps ...
+            #         in which case we keep only the first one found)
+            if app in app_dict:
+                logger.warning("Duplicate app %s found between apps catalog %s and %s" % (app, apps_catalog_id, app_dict[app]['repository']))
+                continue
+
+            info['repository'] = apps_catalog_id
+            app_dict[app] = info
+
+    return app_dict
+
+#
+# ############################### #
+#        Small utilities          #
+# ############################### #
+#
 
 
 def is_true(arg):

src/yunohost/data_migrations/0010_migrate_to_apps_json.py

@@ -1,48 +1,13 @@
-import os
-
 from moulinette.utils.log import getActionLogger
-from yunohost.app import app_fetchlist, app_removelist, _read_appslist_list, APPSLISTS_JSON
 from yunohost.tools import Migration
 
 logger = getActionLogger('yunohost.migration')
 
-BASE_CONF_PATH = '/home/yunohost.conf'
-BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
-APPSLISTS_BACKUP = os.path.join(BACKUP_CONF_DIR, "appslist_before_migration_to_unified_list.json")
-
 
 class MyMigration(Migration):
 
-    "Migrate from official.json to apps.json"
+    "Migrate from official.json to apps.json (outdated, replaced by migration 13)"
 
     def run(self):
-
+        logger.info("This migration is oudated and doesn't do anything anymore. The migration 13 will handle this instead.")
-        # Backup current app list json
+        pass
-        os.system("cp %s %s" % (APPSLISTS_JSON, APPSLISTS_BACKUP))
-
-        # Remove all the deprecated lists
-        lists_to_remove = [
-            "app.yunohost.org/list.json",       # Old list on old installs, alias to official.json
-            "app.yunohost.org/official.json",
-            "app.yunohost.org/community.json",
-            "labriqueinter.net/apps/labriqueinternet.json",
-            "labriqueinter.net/internetcube.json"
-        ]
-        try:
-            appslists = _read_appslist_list()
-            for appslist, infos in appslists.items():
-                if infos["url"].split("//")[-1] in lists_to_remove:
-                    app_removelist(name=appslist)
-
-            # Replace by apps.json list
-            app_fetchlist(name="yunohost",
-                          url="https://app.yunohost.org/apps.json")
-        except Exception:
-            if os.path.exists(APPSLISTS_BACKUP):
-                os.system("cp %s %s" % (APPSLISTS_BACKUP, APPSLISTS_JSON))
-            raise
-        else:
-            if os.path.exists(APPSLISTS_BACKUP):
-                os.remove(APPSLISTS_BACKUP)
-
-

src/yunohost/data_migrations/0013_futureproof_apps_catalog_system.py

@@ -0,0 +1,46 @@
+
+import os
+import shutil
+
+from moulinette.utils.log import getActionLogger
+from moulinette.utils.filesystem import read_json
+
+from yunohost.tools import Migration
+from yunohost.app import (_initialize_apps_catalog_system,
+                          _update_apps_catalog,
+                          APPS_CATALOG_CACHE,
+                          APPS_CATALOG_CONF)
+
+logger = getActionLogger('yunohost.migration')
+
+LEGACY_APPS_CATALOG_CONF = '/etc/yunohost/appslists.json'
+LEGACY_APPS_CATALOG_CONF_BACKUP = LEGACY_APPS_CATALOG_CONF + ".old"
+
+
+class MyMigration(Migration):
+
+    "Migrate to the new future-proof apps catalog system"
+
+    def migrate(self):
+
+        if not os.path.exists(LEGACY_APPS_CATALOG_CONF):
+            logger.info("No need to do anything")
+
+        # Destroy old lecacy cache
+        if os.path.exists(APPS_CATALOG_CACHE):
+            shutil.rmtree(APPS_CATALOG_CACHE)
+
+        # Backup the legacy file
+        try:
+            legacy_catalogs = read_json(LEGACY_APPS_CATALOG_CONF)
+            # If there's only one catalog, we assume it's just the old official catalog
+            # Otherwise, warn the (power-?)users that they should migrate their old catalogs manually
+            if len(legacy_catalogs) > 1:
+                logger.warning("It looks like you had additional apps_catalog in the configuration file %s! YunoHost now uses %s instead, but it won't migrate your custom apps_catalog. You should do this manually. The old file has been backuped in %s." % (LEGACY_APPS_CATALOG_CONF, APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP))
+        except Exception as e:
+            logger.warning("Unable to parse the legacy conf %s (error : %s) ... migrating anyway" % (LEGACY_APPS_CATALOG_CONF, str(e)))
+
+        os.rename(LEGACY_APPS_CATALOG_CONF, LEGACY_APPS_CATALOG_CONF_BACKUP)
+
+        _initialize_apps_catalog_system()
+        _update_apps_catalog()

src/yunohost/tests/test_appscatalog.py

@@ -0,0 +1,358 @@
+import os
+import pytest
+import requests
+import requests_mock
+import glob
+import shutil
+
+from moulinette import m18n
+from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml, mkdir
+
+from yunohost.utils.error import YunohostError
+from yunohost.app import (_initialize_apps_catalog_system,
+                          _read_apps_catalog_list,
+                          _update_apps_catalog,
+                          _actual_apps_catalog_api_url,
+                          _load_apps_catalog,
+                          logger,
+                          APPS_CATALOG_CACHE,
+                          APPS_CATALOG_CONF,
+                          APPS_CATALOG_CRON_PATH,
+                          APPS_CATALOG_API_VERSION,
+                          APPS_CATALOG_DEFAULT_URL)
+
+APPS_CATALOG_DEFAULT_URL_FULL = _actual_apps_catalog_api_url(APPS_CATALOG_DEFAULT_URL)
+CRON_FOLDER, CRON_NAME = APPS_CATALOG_CRON_PATH.rsplit("/", 1)
+
+DUMMY_APP_CATALOG = """{
+   "foo": {"id": "foo", "level": 4},
+   "bar": {"id": "bar", "level": 7}
+}
+"""
+
+class AnyStringWith(str):
+    def __eq__(self, other):
+        return self in other
+
+def setup_function(function):
+
+    # Clear apps catalog cache
+    shutil.rmtree(APPS_CATALOG_CACHE, ignore_errors=True)
+
+    # Clear apps_catalog cron
+    if os.path.exists(APPS_CATALOG_CRON_PATH):
+        os.remove(APPS_CATALOG_CRON_PATH)
+
+    # Clear apps_catalog conf
+    if os.path.exists(APPS_CATALOG_CONF):
+        os.remove(APPS_CATALOG_CONF)
+
+
+def teardown_function(function):
+
+    # Clear apps catalog cache
+    # Otherwise when using apps stuff after running the test,
+    # we'll still have the dummy unusable list
+    shutil.rmtree(APPS_CATALOG_CACHE, ignore_errors=True)
+
+
+def cron_job_is_there():
+    r = os.system("run-parts -v --test %s | grep %s" % (CRON_FOLDER, CRON_NAME))
+    return r == 0
+
+#
+# ################################################
+#
+
+
+def test_apps_catalog_init(mocker):
+
+    # Cache is empty
+    assert not glob.glob(APPS_CATALOG_CACHE + "/*")
+    # Conf doesn't exist yet
+    assert not os.path.exists(APPS_CATALOG_CONF)
+    # Conf doesn't exist yet
+    assert not os.path.exists(APPS_CATALOG_CRON_PATH)
+
+    # Initialize ...
+    mocker.spy(m18n, "n")
+    _initialize_apps_catalog_system()
+    m18n.n.assert_any_call('apps_catalog_init_success')
+
+    # Then there's a cron enabled
+    assert cron_job_is_there()
+
+    # And a conf with at least one list
+    assert os.path.exists(APPS_CATALOG_CONF)
+    apps_catalog_list = _read_apps_catalog_list()
+    assert len(apps_catalog_list)
+
+    # Cache is expected to still be empty though
+    # (if we did update the apps_catalog during init,
+    # we couldn't differentiate easily exceptions
+    # related to lack of network connectivity)
+    assert not glob.glob(APPS_CATALOG_CACHE + "/*")
+
+
+def test_apps_catalog_emptylist():
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    # Let's imagine somebody removed the default apps catalog because uh idk they dont want to use our default apps catalog
+    os.system("rm %s" % APPS_CATALOG_CONF)
+    os.system("touch %s" % APPS_CATALOG_CONF)
+
+    apps_catalog_list = _read_apps_catalog_list()
+    assert not len(apps_catalog_list)
+
+
+def test_apps_catalog_update_success(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    # Cache is empty
+    assert not glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    # Update
+    with requests_mock.Mocker() as m:
+
+        _actual_apps_catalog_api_url,
+        # Mock the server response with a dummy apps catalog
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+
+        mocker.spy(m18n, "n")
+        _update_apps_catalog()
+        m18n.n.assert_any_call("apps_catalog_updating")
+        m18n.n.assert_any_call("apps_catalog_update_success")
+
+    # Cache shouldn't be empty anymore empty
+    assert glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    app_dict = _load_apps_catalog()
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+
+def test_apps_catalog_update_404(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    with requests_mock.Mocker() as m:
+
+        # 404 error
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL,
+                       status_code=404)
+
+        with pytest.raises(YunohostError):
+            mocker.spy(m18n, "n")
+            _update_apps_catalog()
+            m18n.n.assert_any_call("apps_catalog_failed_to_download")
+
+def test_apps_catalog_update_timeout(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    with requests_mock.Mocker() as m:
+
+        # Timeout
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL,
+                       exc=requests.exceptions.ConnectTimeout)
+
+        with pytest.raises(YunohostError):
+            mocker.spy(m18n, "n")
+            _update_apps_catalog()
+            m18n.n.assert_any_call("apps_catalog_failed_to_download")
+
+
+def test_apps_catalog_update_sslerror(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    with requests_mock.Mocker() as m:
+
+        # SSL error
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL,
+                       exc=requests.exceptions.SSLError)
+
+        with pytest.raises(YunohostError):
+            mocker.spy(m18n, "n")
+            _update_apps_catalog()
+            m18n.n.assert_any_call("apps_catalog_failed_to_download")
+
+
+def test_apps_catalog_update_corrupted(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    with requests_mock.Mocker() as m:
+
+        # Corrupted json
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL,
+                       text=DUMMY_APP_CATALOG[:-2])
+
+        with pytest.raises(YunohostError):
+            mocker.spy(m18n, "n")
+            _update_apps_catalog()
+            m18n.n.assert_any_call("apps_catalog_failed_to_download")
+
+
+def test_apps_catalog_load_with_empty_cache(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    # Cache is empty
+    assert not glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    # Update
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a dummy apps catalog
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+
+        # Try to load the apps catalog
+        # This should implicitly trigger an update in the background
+        mocker.spy(m18n, "n")
+        app_dict = _load_apps_catalog()
+        m18n.n.assert_any_call("apps_catalog_obsolete_cache")
+        m18n.n.assert_any_call("apps_catalog_update_success")
+
+
+    # Cache shouldn't be empty anymore empty
+    assert glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+
+def test_apps_catalog_load_with_conflicts_between_lists(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    conf = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL},
+            {"id": "default2", "url": APPS_CATALOG_DEFAULT_URL.replace("yunohost.org", "yolohost.org")}]
+
+    write_to_yaml(APPS_CATALOG_CONF, conf)
+
+    # Update
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a dummy apps catalog
+        # + the same apps catalog for the second list
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL.replace("yunohost.org", "yolohost.org"), text=DUMMY_APP_CATALOG)
+
+        # Try to load the apps catalog
+        # This should implicitly trigger an update in the background
+        mocker.spy(logger, "warning")
+        app_dict = _load_apps_catalog()
+        logger.warning.assert_any_call(AnyStringWith("Duplicate"))
+
+    # Cache shouldn't be empty anymore empty
+    assert glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+
+def test_apps_catalog_load_with_oudated_api_version(mocker):
+
+    # Initialize ...
+    _initialize_apps_catalog_system()
+
+    # Update
+    with requests_mock.Mocker() as m:
+
+        mocker.spy(m18n, "n")
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+        _update_apps_catalog()
+
+    # Cache shouldn't be empty anymore empty
+    assert glob.glob(APPS_CATALOG_CACHE + "/*")
+
+    # Tweak the cache to replace the from_api_version with a different one
+    for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"):
+        cache_json = read_json(cache_file)
+        assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION
+        cache_json["from_api_version"] = 0
+        write_to_json(cache_file, cache_json)
+
+    # Update
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a dummy apps catalog
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+
+        mocker.spy(m18n, "n")
+        app_dict = _load_apps_catalog()
+        m18n.n.assert_any_call("apps_catalog_update_success")
+
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+    # Check that we indeed have the new api number in cache
+    for cache_file in glob.glob(APPS_CATALOG_CACHE + "/*"):
+        cache_json = read_json(cache_file)
+        assert cache_json["from_api_version"] == APPS_CATALOG_API_VERSION
+
+
+
+def test_apps_catalog_migrate_legacy_explicitly():
+
+    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
+    mkdir(APPS_CATALOG_CACHE, 0o750, parents=True)
+    open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"foo":{}, "bar": {}}')
+    open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron")
+
+    from yunohost.tools import _get_migration_by_name
+    migration = _get_migration_by_name("futureproof_apps_catalog_system")
+
+    with requests_mock.Mocker() as m:
+
+        # Mock the server response with a dummy apps catalog
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+        migration.migrate()
+
+    # Old conf shouldnt be there anymore (got renamed to .old)
+    assert not os.path.exists("/etc/yunohost/appslists.json")
+    # Old cache should have been removed
+    assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json")
+    # Cron should have been changed
+    assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read()
+    assert cron_job_is_there()
+
+    # Reading the apps_catalog should work
+    app_dict = _load_apps_catalog()
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+
+def test_apps_catalog_migrate_legacy_implicitly():
+
+    open("/etc/yunohost/appslists.json", "w").write('{"yunohost": {"yolo":"swag"}}')
+    mkdir(APPS_CATALOG_CACHE, 0o750, parents=True)
+    open(APPS_CATALOG_CACHE+"/yunohost_old.json", "w").write('{"old_foo":{}, "old_bar": {}}')
+    open(APPS_CATALOG_CRON_PATH, "w").write("# Some old cron")
+
+    with requests_mock.Mocker() as m:
+        m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG)
+        app_dict = _load_apps_catalog()
+
+    assert "foo" in app_dict.keys()
+    assert "bar" in app_dict.keys()
+
+    # Old conf shouldnt be there anymore (got renamed to .old)
+    assert not os.path.exists("/etc/yunohost/appslists.json")
+    # Old cache should have been removed
+    assert not os.path.exists(APPS_CATALOG_CACHE+"/yunohost_old.json")
+    # Cron should have been changed
+    assert "/bin/bash" in open(APPS_CATALOG_CRON_PATH, "r").read()
+    assert cron_job_is_there()
+

src/yunohost/tests/test_appslist.py

@@ -1,389 +0,0 @@
-import os
-import pytest
-import requests
-import requests_mock
-import glob
-import time
-
-from yunohost.utils.error import YunohostError
-
-from yunohost.app import app_fetchlist, app_removelist, app_listlists, _using_legacy_appslist_system, _migrate_appslist_system, _register_new_appslist
-
-URL_OFFICIAL_APP_LIST = "https://app.yunohost.org/official.json"
-REPO_PATH = '/var/cache/yunohost/repo'
-APPSLISTS_JSON = '/etc/yunohost/appslists.json'
-
-
-def setup_function(function):
-
-    # Clear all appslist
-    files = glob.glob(REPO_PATH + "/*")
-    for f in files:
-        os.remove(f)
-
-    # Clear appslist crons
-    files = glob.glob("/etc/cron.d/yunohost-applist-*")
-    for f in files:
-        os.remove(f)
-
-    if os.path.exists("/etc/cron.daily/yunohost-fetch-appslists"):
-        os.remove("/etc/cron.daily/yunohost-fetch-appslists")
-
-    if os.path.exists(APPSLISTS_JSON):
-        os.remove(APPSLISTS_JSON)
-
-
-def teardown_function(function):
-    pass
-
-
-def cron_job_is_there():
-    r = os.system("run-parts -v --test /etc/cron.daily/ | grep yunohost-fetch-appslists")
-    return r == 0
-
-
-#
-# Test listing of appslists and registering of appslists                      #
-#
-
-
-def test_appslist_list_empty():
-    """
-    Calling app_listlists() with no registered list should return empty dict
-    """
-
-    assert app_listlists() == {}
-
-
-def test_appslist_list_register():
-    """
-    Register a new list
-    """
-
-    # Assume we're starting with an empty app list
-    assert app_listlists() == {}
-
-    # Register a new dummy list
-    _register_new_appslist("https://lol.com/appslist.json", "dummy")
-
-    appslist_dict = app_listlists()
-    assert "dummy" in appslist_dict.keys()
-    assert appslist_dict["dummy"]["url"] == "https://lol.com/appslist.json"
-
-    assert cron_job_is_there()
-
-
-def test_appslist_list_register_conflict_name():
-    """
-    Attempt to register a new list with conflicting name
-    """
-
-    _register_new_appslist("https://lol.com/appslist.json", "dummy")
-    with pytest.raises(YunohostError):
-        _register_new_appslist("https://lol.com/appslist2.json", "dummy")
-
-    appslist_dict = app_listlists()
-
-    assert "dummy" in appslist_dict.keys()
-    assert "dummy2" not in appslist_dict.keys()
-
-
-def test_appslist_list_register_conflict_url():
-    """
-    Attempt to register a new list with conflicting url
-    """
-
-    _register_new_appslist("https://lol.com/appslist.json", "dummy")
-    with pytest.raises(YunohostError):
-        _register_new_appslist("https://lol.com/appslist.json", "plopette")
-
-    appslist_dict = app_listlists()
-
-    assert "dummy" in appslist_dict.keys()
-    assert "plopette" not in appslist_dict.keys()
-
-
-#
-# Test fetching of appslists                                                 #
-#
-
-
-def test_appslist_fetch():
-    """
-    Do a fetchlist and test the .json got updated.
-    """
-    assert app_listlists() == {}
-
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-
-    with requests_mock.Mocker() as m:
-
-        # Mock the server response with a valid (well, empty, yep) json
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
-
-        official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
-        app_fetchlist()
-        new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
-
-    assert new_official_lastUpdate > official_lastUpdate
-
-
-def test_appslist_fetch_single_appslist():
-    """
-    Register several lists but only fetch one. Check only one got updated.
-    """
-
-    assert app_listlists() == {}
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-    _register_new_appslist("https://lol.com/appslist.json", "dummy")
-
-    time.sleep(1)
-
-    with requests_mock.Mocker() as m:
-
-        # Mock the server response with a valid (well, empty, yep) json
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ }')
-
-        official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
-        dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
-        app_fetchlist(name="yunohost")
-        new_official_lastUpdate = app_listlists()["yunohost"]["lastUpdate"]
-        new_dummy_lastUpdate = app_listlists()["dummy"]["lastUpdate"]
-
-    assert new_official_lastUpdate > official_lastUpdate
-    assert new_dummy_lastUpdate == dummy_lastUpdate
-
-
-def test_appslist_fetch_unknownlist():
-    """
-    Attempt to fetch an unknown list
-    """
-
-    assert app_listlists() == {}
-
-    with pytest.raises(YunohostError):
-        app_fetchlist(name="swag")
-
-
-def test_appslist_fetch_url_but_no_name():
-    """
-    Do a fetchlist with url given, but no name given
-    """
-
-    with pytest.raises(YunohostError):
-        app_fetchlist(url=URL_OFFICIAL_APP_LIST)
-
-
-def test_appslist_fetch_badurl():
-    """
-    Do a fetchlist with a bad url
-    """
-
-    app_fetchlist(url="https://not.a.valid.url/plop.json", name="plop")
-
-
-def test_appslist_fetch_badfile():
-    """
-    Do a fetchlist and mock a response with a bad json
-    """
-    assert app_listlists() == {}
-
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-
-    with requests_mock.Mocker() as m:
-
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST, text='{ not json lol }')
-
-        app_fetchlist()
-
-
-def test_appslist_fetch_404():
-    """
-    Do a fetchlist and mock a 404 response
-    """
-    assert app_listlists() == {}
-
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-
-    with requests_mock.Mocker() as m:
-
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST, status_code=404)
-
-        app_fetchlist()
-
-
-def test_appslist_fetch_sslerror():
-    """
-    Do a fetchlist and mock an SSL error
-    """
-    assert app_listlists() == {}
-
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-
-    with requests_mock.Mocker() as m:
-
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST,
-                       exc=requests.exceptions.SSLError)
-
-        app_fetchlist()
-
-
-def test_appslist_fetch_timeout():
-    """
-    Do a fetchlist and mock a timeout
-    """
-    assert app_listlists() == {}
-
-    _register_new_appslist(URL_OFFICIAL_APP_LIST, "yunohost")
-
-    with requests_mock.Mocker() as m:
-
-        m.register_uri("GET", URL_OFFICIAL_APP_LIST,
-                       exc=requests.exceptions.ConnectTimeout)
-
-        app_fetchlist()
-
-
-#
-# Test remove of appslist                                                    #
-#
-
-
-def test_appslist_remove():
-    """
-    Register a new appslist, then remove it
-    """
-
-    # Assume we're starting with an empty app list
-    assert app_listlists() == {}
-
-    # Register a new dummy list
-    _register_new_appslist("https://lol.com/appslist.json", "dummy")
-    app_removelist("dummy")
-
-    # Should end up with no list registered
-    assert app_listlists() == {}
-
-
-def test_appslist_remove_unknown():
-    """
-    Attempt to remove an unknown list
-    """
-
-    with pytest.raises(YunohostError):
-        app_removelist("dummy")
-
-
-#
-# Test migration from legacy appslist system                                 #
-#
-
-
-def add_legacy_cron(name, url):
-    with open("/etc/cron.d/yunohost-applist-%s" % name, "w") as f:
-        f.write('00 00 * * * root yunohost app fetchlist -u %s -n %s > /dev/null 2>&1\n' % (url, name))
-
-
-def test_appslist_check_using_legacy_system_testFalse():
-    """
-    If no legacy cron job is there, the check should return False
-    """
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    assert _using_legacy_appslist_system() is False
-
-
-def test_appslist_check_using_legacy_system_testTrue():
-    """
-    If there's a legacy cron job, the check should return True
-    """
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
-    assert _using_legacy_appslist_system() is True
-
-
-def test_appslist_system_migration():
-    """
-    Test that legacy cron jobs get migrated correctly when calling app_listlists
-    """
-
-    # Start with no legacy cron, no appslist registered
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    assert app_listlists() == {}
-    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
-
-    # Add a few legacy crons
-    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
-    add_legacy_cron("dummy", "https://swiggitty.swaggy.lol/yolo.json")
-
-    # Migrate
-    assert _using_legacy_appslist_system() is True
-    _migrate_appslist_system()
-    assert _using_legacy_appslist_system() is False
-
-    # No legacy cron job should remain
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-
-    # Check they are in app_listlists anyway
-    appslist_dict = app_listlists()
-    assert "yunohost" in appslist_dict.keys()
-    assert appslist_dict["yunohost"]["url"] == "https://app.yunohost.org/official.json"
-    assert "dummy" in appslist_dict.keys()
-    assert appslist_dict["dummy"]["url"] == "https://swiggitty.swaggy.lol/yolo.json"
-
-    assert cron_job_is_there()
-
-
-def test_appslist_system_migration_badcron():
-    """
-    Test the migration on a bad legacy cron (no url found inside cron job)
-    """
-
-    # Start with no legacy cron, no appslist registered
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    assert app_listlists() == {}
-    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
-
-    # Add a "bad" legacy cron
-    add_legacy_cron("wtflist", "ftp://the.fuck.is.this")
-
-    # Migrate
-    assert _using_legacy_appslist_system() is True
-    _migrate_appslist_system()
-    assert _using_legacy_appslist_system() is False
-
-    # No legacy cron should remain, but it should be backuped in /etc/yunohost
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    assert os.path.exists("/etc/yunohost/wtflist.oldlist.bkp")
-
-    # Appslist should still be empty
-    assert app_listlists() == {}
-
-
-def test_appslist_system_migration_conflict():
-    """
-    Test migration of conflicting cron job (in terms of url)
-    """
-
-    # Start with no legacy cron, no appslist registered
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-    assert app_listlists() == {}
-    assert not os.path.exists("/etc/cron.daily/yunohost-fetch-appslists")
-
-    # Add a few legacy crons
-    add_legacy_cron("yunohost", "https://app.yunohost.org/official.json")
-    add_legacy_cron("dummy", "https://app.yunohost.org/official.json")
-
-    # Migrate
-    assert _using_legacy_appslist_system() is True
-    _migrate_appslist_system()
-    assert _using_legacy_appslist_system() is False
-
-    # No legacy cron job should remain
-    assert glob.glob("/etc/cron.d/yunohost-applist-*") == []
-
-    # Only one among "dummy" and "yunohost" should be listed
-    appslist_dict = app_listlists()
-    assert (len(appslist_dict.keys()) == 1)
-    assert ("dummy" in appslist_dict.keys()) or ("yunohost" in appslist_dict.keys())
-
-    assert cron_job_is_there()

src/yunohost/tools.py

@@ -36,7 +36,8 @@ from moulinette import msignals, m18n
 from moulinette.utils.log import getActionLogger
 from moulinette.utils.process import check_output, call_async_output
 from moulinette.utils.filesystem import read_json, write_to_json, read_yaml, write_to_yaml
-from yunohost.app import app_fetchlist, app_info, app_upgrade, app_ssowatconf, _install_appslist_fetch_cron
+
+from yunohost.app import _update_apps_catalog, app_info, app_upgrade, app_ssowatconf, app_list
 from yunohost.domain import domain_add, domain_list
 from yunohost.dyndns import _dyndns_available, _dyndns_provides
 from yunohost.firewall import firewall_upnp
@@ -350,15 +351,17 @@ def tools_postinstall(operation_logger, domain, password, ignore_dyndns=False,
     # Enable UPnP silently and reload firewall
     firewall_upnp('enable', no_refresh=True)
 
-    # Setup the default apps list with cron job
+    # Initialize the apps catalog system
+    _initialize_apps_catalog_system()
+
+    # Try to update the apps catalog ...
+    # we don't fail miserably if this fails,
+    # because that could be for example an offline installation...
     try:
-        app_fetchlist(name="yunohost",
+        _update_apps_catalog()
-                      url="https://app.yunohost.org/apps.json")
     except Exception as e:
         logger.warning(str(e))
 
-    _install_appslist_fetch_cron()
-
     # Init migrations (skip them, no need to run them on a fresh system)
     _skip_all_migrations()
 
@@ -449,11 +452,10 @@ def tools_update(apps=False, system=False):
 
     upgradable_apps = []
     if apps:
-        logger.info(m18n.n('updating_app_lists'))
         try:
-            app_fetchlist()
+            _update_apps_catalog()
         except YunohostError as e:
-            logger.error(m18n.n('tools_update_failed_to_app_fetchlist'), error=e)
+            logger.error(str(e))
 
         upgradable_apps = list(_list_upgradable_apps())