Turn the i18n key checker into tests for tox

tests/test_i18n_keys.py

@@ -2,23 +2,22 @@
 
 import os
 import re
-import sys
 import glob
 import json
 import yaml
 import subprocess
 
-ignore = [ "password_too_simple_",
+ignore = ["password_too_simple_",
-           "password_listed",
+          "password_listed",
-           "backup_method_",
+          "backup_method_",
-           "backup_applying_method_",
+          "backup_applying_method_",
-           "confirm_app_install_",
+          "confirm_app_install_"]
-        ]
 
 ###############################################################################
 #   Find used keys in python code                                             #
 ###############################################################################
 
+
 def find_expected_string_keys():
 
     # Try to find :
@@ -27,33 +26,40 @@ def find_expected_string_keys():
     p1 = re.compile(r'm18n\.n\(\s*[\"\'](\w+)[\"\']')
     p2 = re.compile(r'YunohostError\([\'\"](\w+)[\'\"]')
 
-    python_files = glob.glob("../src/yunohost/*.py")
+    python_files = glob.glob("src/yunohost/*.py")
-    python_files.extend(glob.glob("../src/yunohost/utils/*.py"))
+    python_files.extend(glob.glob("src/yunohost/utils/*.py"))
-    python_files.extend(glob.glob("../src/yunohost/data_migrations/*.py"))
+    python_files.extend(glob.glob("src/yunohost/data_migrations/*.py"))
-    python_files.extend(glob.glob("../data/hooks/diagnosis/*.py"))
+    python_files.extend(glob.glob("data/hooks/diagnosis/*.py"))
-    python_files.append("../bin/yunohost")
+    python_files.append("bin/yunohost")
 
     for python_file in python_files:
         content = open(python_file).read()
-        yield from p1.findall(content)
+        for m in p1.findall(content):
-        yield from p2.findall(content)
+            if m.endswith("_"):
+                continue
+            yield m
+        for m in p2.findall(content):
+            if m.endswith("_"):
+                continue
+            yield m
 
     # For each diagnosis, try to find strings like "diagnosis_stuff_foo" (c.f. diagnosis summaries)
     # Also we expect to have "diagnosis_description_<name>" for each diagnosis
     p3 = re.compile(r'[\"\'](diagnosis_[a-z]+_\w+)[\"\']')
-    for python_file in glob.glob("../data/hooks/diagnosis/*.py"):
+    for python_file in glob.glob("data/hooks/diagnosis/*.py"):
         content = open(python_file).read()
-        yield from p3.findall(content)
+        for m in p3.findall(content):
+            yield m
         yield "diagnosis_description_" + os.path.basename(python_file)[:-3].split("-")[-1]
 
     # For each migration, expect to find "migration_description_<name>"
-    for path in glob.glob("../src/yunohost/data_migrations/*.py"):
+    for path in glob.glob("src/yunohost/data_migrations/*.py"):
         if "__init__" in path:
             continue
         yield "migration_description_" + os.path.basename(path)[:-3]
 
     # For each default service, expect to find "service_description_<name>"
-    for service, info in yaml.safe_load(open("../data/templates/yunohost/services.yml")).items():
+    for service, info in yaml.safe_load(open("data/templates/yunohost/services.yml")).items():
         if info is None:
             continue
         yield "service_description_" + service
@@ -61,23 +67,25 @@ def find_expected_string_keys():
     # For all unit operations, expect to find "log_<name>"
     # A unit operation is created either using the @is_unit_operation decorator
     # or using OperationLogger(
-    cmd = "grep -hr '@is_unit_operation' ../src/yunohost/ -A3 2>/dev/null | grep '^def' | sed -E 's@^def (\w+)\(.*@\\1@g'"
+    cmd = "grep -hr '@is_unit_operation' src/yunohost/ -A3 2>/dev/null | grep '^def' | sed -E 's@^def (\\w+)\\(.*@\\1@g'"
-    for funcname in subprocess.check_output(cmd, shell=True).decode("utf-8").split("\n"):
+    for funcname in subprocess.check_output(cmd, shell=True).decode("utf-8").strip().split("\n"):
-        yield "log_"+funcname
+        yield "log_" + funcname
 
     p4 = re.compile(r"OperationLogger\([\"\'](\w+)[\"\']")
     for python_file in python_files:
         content = open(python_file).read()
-        yield from ("log_"+match for match in p4.findall(content))
+        for m in ("log_" + match for match in p4.findall(content)):
+            yield m
 
     # Global settings descriptions
     # Will be on a line like : ("service.ssh.allow_deprecated_dsa_hostkey", {"type": "bool", ...
     p5 = re.compile(r" \([\"\'](\w[\w\.]+)[\"\'],")
-    content = open("../src/yunohost/settings.py").read()
+    content = open("src/yunohost/settings.py").read()
-    yield from ("global_settings_setting_"+s.replace(".", "_") for s in p5.findall(content))
+    for m in ("global_settings_setting_" + s.replace(".", "_") for s in p5.findall(content)):
+        yield m
 
     # Keys for the actionmap ...
-    for category in yaml.load(open("../data/actionsmap/yunohost.yml")).values():
+    for category in yaml.load(open("data/actionsmap/yunohost.yml")).values():
         if "actions" not in category.keys():
             continue
         for action in category["actions"].values():
@@ -98,41 +106,54 @@ def find_expected_string_keys():
                 if "help" in extra:
                     yield extra["help"]
 
-expected_string_keys = set(find_expected_string_keys())
+    # Hardcoded expected keys ...
+    yield "admin_password"  # Not sure that's actually used nowadays...
 
-expected_string_keys.add("admin_password")
+    for method in ["tar", "copy", "borg", "custom"]:
+        yield "backup_applying_method_%s" % method
+        yield "backup_method_%s_finished" % method
+
+    for level in ["danger", "thirdparty", "warning"]:
+        yield "confirm_app_install_%s" % level
+
+    for errortype in ["bad_status_code", "connection_error", "timeout"]:
+        yield "diagnosis_http_%s" % errortype
+
+    yield "password_listed"
+    for i in [1, 2, 3, 4]:
+        yield "password_too_simple_%s" % i
 
 ###############################################################################
 #   Load en locale json keys                                                  #
 ###############################################################################
 
-en_locale_file = "../locales/en.json"
-with open(en_locale_file) as f:
-    en_locale_json = json.loads(f.read())
 
-en_locale_keys = set(en_locale_json.keys())
+def keys_defined_for_en():
+    return json.loads(open("locales/en.json").read()).keys()
 
 ###############################################################################
 #   Compare keys used and keys defined                                        #
 ###############################################################################
 
-keys_used_but_not_defined = expected_string_keys.difference(en_locale_keys)
-keys_defined_but_not_used = en_locale_keys.difference(expected_string_keys)
 
-if len(keys_used_but_not_defined) != 0:
+expected_string_keys = set(find_expected_string_keys())
-    print("> Error ! Those keys are used in some files but not defined :")
+keys_defined = set(keys_defined_for_en())
-    for key in sorted(keys_used_but_not_defined):
-        if any(key.startswith(i) for i in ignore):
-            continue
-        print("   - %s" % key)
-
-if len(keys_defined_but_not_used) != 0:
-    print("> Warning ! Those keys are defined but seems unused :")
-    for key in sorted(keys_defined_but_not_used):
-        if any(key.startswith(i) for i in ignore):
-            continue
-        print("   - %s" % key)
 
 
-if len(keys_used_but_not_defined) != 0 or len(keys_defined_but_not_used) != 0:
+def test_undefined_i18n_keys():
-    sys.exit(1)
+    undefined_keys = expected_string_keys.difference(keys_defined)
+    undefined_keys = sorted(undefined_keys)
+
+    if undefined_keys:
+        raise Exception("Those i18n keys should be defined in en.json:\n"
+                        "    - " + "\n    - ".join(undefined_keys))
+
+
+def test_unused_i18n_keys():
+
+    unused_keys = keys_defined.difference(expected_string_keys)
+    unused_keys = sorted(unused_keys)
+
+    if unused_keys:
+        raise Exception("Those i18n keys appears unused:\n"
+                        "    - " + "\n    - ".join(unused_keys))