mirror of
https://github.com/YunoHost/yunohost.git
synced 2024-09-03 20:06:10 +02:00
Improve / rework script meant to check that all i18n keys used in code are effectively defined
This commit is contained in:
Alexandre Aubin
parent
c0fc60aa5f
commit
9732bbab58
2 changed files with 138 additions and 101 deletions
|
|
@ -1,101 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
import glob
|
||||
import json
|
||||
import yaml
|
||||
|
||||
ignore = [ "service_description_",
|
||||
"migration_description_",
|
||||
"global_settings_setting_",
|
||||
"password_too_simple_",
|
||||
"password_listed",
|
||||
"backup_method_",
|
||||
"backup_applying_method_",
|
||||
"confirm_app_install_",
|
||||
"log_",
|
||||
]
|
||||
|
||||
###############################################################################
|
||||
# Find used keys in python code #
|
||||
###############################################################################
|
||||
|
||||
# This regex matches « foo » in patterns like « m18n.n( "foo" »
|
||||
p1 = re.compile(r'm18n\.n\(\s*[\"\']([a-zA-Z0-9_]+)[\"\']')
|
||||
p2 = re.compile(r'YunohostError\([\'\"]([a-zA-Z0-9_]+)[\'\"]')
|
||||
|
||||
python_files = glob.glob("../src/yunohost/*.py")
|
||||
python_files.extend(glob.glob("../src/yunohost/utils/*.py"))
|
||||
python_files.extend(glob.glob("../src/yunohost/data_migrations/*.py"))
|
||||
python_files.append("../bin/yunohost")
|
||||
|
||||
python_keys = set()
|
||||
for python_file in python_files:
|
||||
content = open(python_file).read()
|
||||
for match in p1.findall(content):
|
||||
python_keys.add(match)
|
||||
for match in p2.findall(content):
|
||||
python_keys.add(match)
|
||||
|
||||
###############################################################################
|
||||
# Find keys used in actionmap #
|
||||
###############################################################################
|
||||
|
||||
actionmap_keys = set()
|
||||
actionmap = yaml.load(open("../data/actionsmap/yunohost.yml"))
|
||||
for _, category in actionmap.items():
|
||||
if "actions" not in category.keys():
|
||||
continue
|
||||
for _, action in category["actions"].items():
|
||||
if "arguments" not in action.keys():
|
||||
continue
|
||||
for _, argument in action["arguments"].items():
|
||||
if "extra" not in argument.keys():
|
||||
continue
|
||||
if "password" in argument["extra"]:
|
||||
actionmap_keys.add(argument["extra"]["password"])
|
||||
if "ask" in argument["extra"]:
|
||||
actionmap_keys.add(argument["extra"]["ask"])
|
||||
if "comment" in argument["extra"]:
|
||||
actionmap_keys.add(argument["extra"]["comment"])
|
||||
if "pattern" in argument["extra"]:
|
||||
actionmap_keys.add(argument["extra"]["pattern"][1])
|
||||
if "help" in argument["extra"]:
|
||||
print argument["extra"]["help"]
|
||||
|
||||
actionmap_keys.add("admin_password")
|
||||
|
||||
###############################################################################
|
||||
# Load en locale json keys #
|
||||
###############################################################################
|
||||
|
||||
en_locale_file = "../locales/en.json"
|
||||
with open(en_locale_file) as f:
|
||||
en_locale_json = json.loads(f.read())
|
||||
|
||||
en_locale_keys = set(en_locale_json.keys())
|
||||
|
||||
###############################################################################
|
||||
# Compare keys used and keys defined #
|
||||
###############################################################################
|
||||
|
||||
used_keys = python_keys.union(actionmap_keys)
|
||||
|
||||
keys_used_but_not_defined = used_keys.difference(en_locale_keys)
|
||||
keys_defined_but_not_used = en_locale_keys.difference(used_keys)
|
||||
|
||||
if len(keys_used_but_not_defined) != 0:
|
||||
print "> Error ! Those keys are used in some files but not defined :"
|
||||
for key in sorted(keys_used_but_not_defined):
|
||||
if any(key.startswith(i) for i in ignore):
|
||||
continue
|
||||
print " - %s" % key
|
||||
|
||||
if len(keys_defined_but_not_used) != 0:
|
||||
print "> Warning ! Those keys are defined but seems unused :"
|
||||
for key in sorted(keys_defined_but_not_used):
|
||||
if any(key.startswith(i) for i in ignore):
|
||||
continue
|
||||
print " - %s" % key
|
||||
|
||||
|
||||
138
tests/check_m18nkeys.py
Normal file
138
tests/check_m18nkeys.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
import json
|
||||
import yaml
|
||||
import subprocess
|
||||
|
||||
ignore = [ "password_too_simple_",
|
||||
"password_listed",
|
||||
"backup_method_",
|
||||
"backup_applying_method_",
|
||||
"confirm_app_install_",
|
||||
]
|
||||
|
||||
###############################################################################
|
||||
# Find used keys in python code #
|
||||
###############################################################################
|
||||
|
||||
def find_expected_string_keys():
|
||||
|
||||
# Try to find :
|
||||
# m18n.n( "foo"
|
||||
# YunohostError("foo"
|
||||
p1 = re.compile(r'm18n\.n\(\s*[\"\'](\w+)[\"\']')
|
||||
p2 = re.compile(r'YunohostError\([\'\"](\w+)[\'\"]')
|
||||
|
||||
python_files = glob.glob("../src/yunohost/*.py")
|
||||
python_files.extend(glob.glob("../src/yunohost/utils/*.py"))
|
||||
python_files.extend(glob.glob("../src/yunohost/data_migrations/*.py"))
|
||||
python_files.extend(glob.glob("../data/hooks/diagnosis/*.py"))
|
||||
python_files.append("../bin/yunohost")
|
||||
|
||||
for python_file in python_files:
|
||||
content = open(python_file).read()
|
||||
yield from p1.findall(content)
|
||||
yield from p2.findall(content)
|
||||
|
||||
# For each diagnosis, try to find strings like "diagnosis_stuff_foo" (c.f. diagnosis summaries)
|
||||
# Also we expect to have "diagnosis_description_<name>" for each diagnosis
|
||||
p3 = re.compile(r'[\"\'](diagnosis_[a-z]+_\w+)[\"\']')
|
||||
for python_file in glob.glob("../data/hooks/diagnosis/*.py"):
|
||||
content = open(python_file).read()
|
||||
yield from p3.findall(content)
|
||||
yield "diagnosis_description_" + os.path.basename(python_file)[:-3].split("-")[-1]
|
||||
|
||||
# For each migration, expect to find "migration_description_<name>"
|
||||
for path in glob.glob("../src/yunohost/data_migrations/*.py"):
|
||||
if "__init__" in path:
|
||||
continue
|
||||
yield "migration_description_" + os.path.basename(path)[:-3]
|
||||
|
||||
# For each default service, expect to find "service_description_<name>"
|
||||
for service, info in yaml.safe_load(open("../data/templates/yunohost/services.yml")).items():
|
||||
if info is None:
|
||||
continue
|
||||
yield "service_description_" + service
|
||||
|
||||
# For all unit operations, expect to find "log_<name>"
|
||||
# A unit operation is created either using the @is_unit_operation decorator
|
||||
# or using OperationLogger(
|
||||
cmd = "grep -hr '@is_unit_operation' ../src/yunohost/ -A3 2>/dev/null | grep '^def' | sed -E 's@^def (\w+)\(.*@\\1@g'"
|
||||
for funcname in subprocess.check_output(cmd, shell=True).decode("utf-8").split("\n"):
|
||||
yield "log_"+funcname
|
||||
|
||||
p4 = re.compile(r"OperationLogger\([\"\'](\w+)[\"\']")
|
||||
for python_file in python_files:
|
||||
content = open(python_file).read()
|
||||
yield from ("log_"+match for match in p4.findall(content))
|
||||
|
||||
# Global settings descriptions
|
||||
# Will be on a line like : ("service.ssh.allow_deprecated_dsa_hostkey", {"type": "bool", ...
|
||||
p5 = re.compile(r" \([\"\'](\w[\w\.]+)[\"\'],")
|
||||
content = open("../src/yunohost/settings.py").read()
|
||||
yield from ("global_settings_setting_"+s.replace(".", "_") for s in p5.findall(content))
|
||||
|
||||
# Keys for the actionmap ...
|
||||
for category in yaml.load(open("../data/actionsmap/yunohost.yml")).values():
|
||||
if "actions" not in category.keys():
|
||||
continue
|
||||
for action in category["actions"].values():
|
||||
if "arguments" not in action.keys():
|
||||
continue
|
||||
for argument in action["arguments"].values():
|
||||
extra = argument.get("extra")
|
||||
if not extra:
|
||||
continue
|
||||
if "password" in extra:
|
||||
yield extra["password"]
|
||||
if "ask" in extra:
|
||||
yield extra["ask"]
|
||||
if "comment" in extra:
|
||||
yield extra["comment"]
|
||||
if "pattern" in extra:
|
||||
yield extra["pattern"][1]
|
||||
if "help" in extra:
|
||||
yield extra["help"]
|
||||
|
||||
expected_string_keys = set(find_expected_string_keys())
|
||||
|
||||
expected_string_keys.add("admin_password")
|
||||
|
||||
###############################################################################
|
||||
# Load en locale json keys #
|
||||
###############################################################################
|
||||
|
||||
en_locale_file = "../locales/en.json"
|
||||
with open(en_locale_file) as f:
|
||||
en_locale_json = json.loads(f.read())
|
||||
|
||||
en_locale_keys = set(en_locale_json.keys())
|
||||
|
||||
###############################################################################
|
||||
# Compare keys used and keys defined #
|
||||
###############################################################################
|
||||
|
||||
keys_used_but_not_defined = expected_string_keys.difference(en_locale_keys)
|
||||
keys_defined_but_not_used = en_locale_keys.difference(expected_string_keys)
|
||||
|
||||
if len(keys_used_but_not_defined) != 0:
|
||||
print("> Error ! Those keys are used in some files but not defined :")
|
||||
for key in sorted(keys_used_but_not_defined):
|
||||
if any(key.startswith(i) for i in ignore):
|
||||
continue
|
||||
print(" - %s" % key)
|
||||
|
||||
if len(keys_defined_but_not_used) != 0:
|
||||
print("> Warning ! Those keys are defined but seems unused :")
|
||||
for key in sorted(keys_defined_but_not_used):
|
||||
if any(key.startswith(i) for i in ignore):
|
||||
continue
|
||||
print(" - %s" % key)
|
||||
|
||||
|
||||
if len(keys_used_but_not_defined) != 0 or len(keys_defined_but_not_used) != 0:
|
||||
sys.exit(1)
|
||||
Loading…
Add table
Reference in a new issue