From f9437bbd3339020cf46c199ac58c5a8f039850ce Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Tue, 16 May 2017 17:18:07 +0200 Subject: [PATCH] Global settings (#229) * [enh] add base to run tests * [enh] start global settings proto * [mod] has -> exists * [enh] configure actionmap for settings module * [enh] add a default values mecanism * [enh] nicer yaml dump * [mod] DRY * [fix] moulinette doesn't respect positionned arguments * [fix] typo * [mod] don't print when it's not needed * [enh] raise a moulinette exception when the key doesn't exists * [mod] add todo comments * [mod] no more default value * [enh] namespace all global settings commands * [mod] unix way, be quiet except when needed * [fix] forgot to add namespace argument to settings-list * [fix] fail_silently wasn't considered as a cli flag * [mod] refactoring * [enh] remove empty namespace on remove * [enh] print a warning if attempt to list an empty namespace settings * [mod] refactoring * [enh] handle case where I can't open settings file * [enh] handle case where I can't write settings file * [enh] handle case where I can't serialized settings data into yaml * [mod] add error return codes * [enh] start to move to new architecture for settings * [enh] uses a dict instead of a tuple for settings * [mod] no more namespace in settings list * [mod] settings.exists isn't relevant anymore * [mod] settings.remove isn't relevant anymore * [enh] fix settings and switch to json * [enh] adapt settings set * [enh] don't set a key that doesn't exists * [enh] check type of keys before settings them * [enh] start implement default * [enh] handle case where key doesnt exist for default * [enh] i18n for bad key type on settings set * [enh] i18n for bad key type on settings set for enum * [mod] exception for weird situation * [mod] this message isn't used anymore * [enh] i18n for unknown setting key from local settings * [mod] style * [enh] start to work on a reset mecanism * [enh] complain if settings_reset is called without yes * [fix] --yes of reset is a boolean option * [enh] backup old settings before resetting * [fix] bad usage of logger * [enh] backup unknown settings * [enh] move settings description in translations * [enh] add tests for settings * [enh] migrate to pytest assert style * [fix] typo * [doc] add some comments for not explicite part of the code * [mod] possibilities -> choices for uniformised vocabulary * [mod] follow rest semantic * [doc] made namespace usage more explicit * [fix] we don't use namespace key anymore * [enh] make settings_default available in cli * [fix] *really* be Rest semantic * [doc] add docstrings to settings module functions * [enh] reset-all and --full option * [fix] Remove unused global_settings_reset_not_yes --- data/actionsmap/yunohost.yml | 49 ++++++ locales/en.json | 13 ++ src/yunohost/settings.py | 236 ++++++++++++++++++++++++++++ src/yunohost/tests/conftest.py | 1 - src/yunohost/tests/test_settings.py | 166 +++++++++++++++++++ 5 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 src/yunohost/settings.py create mode 100644 src/yunohost/tests/test_settings.py diff --git a/data/actionsmap/yunohost.yml b/data/actionsmap/yunohost.yml index 39c62398c..09b5687f3 100644 --- a/data/actionsmap/yunohost.yml +++ b/data/actionsmap/yunohost.yml @@ -976,6 +976,55 @@ monitor: action_help: Disable server monitoring +############################# +# Settings # +############################# +settings: + category_help: Manage YunoHost global settings + actions: + + ### settings_list() + list: + action_help: list all entries of the settings + api: GET /settings + + ### settings_get() + get: + action_help: get an entry value in the settings + api: GET /settings/ + arguments: + key: + help: Settings key + --full: + help: Show more details + action: store_true + + ### settings_set() + set: + action_help: set an entry value in the settings + api: POST /settings/ + arguments: + key: + help: Settings key + -v: + full: --value + help: new value + extra: + required: True + + ### settings_reset_all() + reset-all: + action_help: reset all settings to their default value + api: DELETE /settings + + ### settings_reset() + reset: + action_help: set an entry value to its default one + api: DELETE /settings/ + arguments: + key: + help: Settings key + ############################# # Service # ############################# diff --git a/locales/en.json b/locales/en.json index 2e85d6d4b..6deffba84 100644 --- a/locales/en.json +++ b/locales/en.json @@ -121,6 +121,19 @@ "firewall_reloaded": "The firewall has been reloaded", "firewall_rules_cmd_failed": "Some firewall rules commands have failed. For more information, see the log.", "format_datetime_short": "%m/%d/%Y %I:%M %p", + "global_settings_bad_choice_for_enum": "Bad value for setting {setting:s}, received {received_type:s}, except {expected_type:s}", + "global_settings_bad_type_for_setting": "Bad type for setting {setting:s}, received {received_type:s}, except {expected_type:s}", + "global_settings_cant_open_settings": "Failed to open settings file, reason: {reason:s}", + "global_settings_cant_serialize_setings": "Failed to serialzed settings data, reason: {reason:s}", + "global_settings_cant_write_settings": "Failed to write settings file, reason: {reason:s}", + "global_settings_key_doesnt_exists": "The key '{settings_key:s}' doesn't exists in the global settings, you can see all the available keys by doing 'yunohost settings list", + "global_settings_reset_success": "Success. Your previous settings have been backuped in {path:s}", + "global_settings_setting_example_bool": "Example boolean option", + "global_settings_setting_example_int": "Example int option", + "global_settings_setting_example_string": "Example string option", + "global_settings_setting_example_enum": "Example enum option", + "global_settings_unknown_type": "Unexpected situation, the setting {setting:s} appears to have the type {unknown_type:s} but it's not a type supported by the system.", + "global_settings_unknown_setting_from_settings_file": "Unknown key in settings: '{setting_key:s}', discarding it and save it in /etc/yunohost/unkown_settings.json", "hook_exec_failed": "Script execution failed: {path:s}", "hook_exec_not_terminated": "Script execution hasn’t terminated: {path:s}", "hook_list_by_invalid": "Invalid property to list hook by", diff --git a/src/yunohost/settings.py b/src/yunohost/settings.py new file mode 100644 index 000000000..c2266a4c9 --- /dev/null +++ b/src/yunohost/settings.py @@ -0,0 +1,236 @@ +import os +import json +import errno + +from datetime import datetime +from collections import OrderedDict + +from moulinette.core import MoulinetteError +from moulinette.utils.log import getActionLogger + +logger = getActionLogger('yunohost.settings') + +SETTINGS_PATH = "/etc/yunohost/settings.json" +SETTINGS_PATH_OTHER_LOCATION = "/etc/yunohost/settings-%s.json" + +# a settings entry is in the form of: +# namespace.subnamespace.name: {type, value, default, description, [choices]} +# choices is only for enum +# the keyname can have as many subnamespace as needed but should have at least +# one level of namespace + +# description is implied from the translated strings +# the key is "global_settings_setting_%s" % key.replace(".", "_") + +# type can be: +# * bool +# * int +# * string +# * enum (in form a python list) + +# we don't store the value in default options +DEFAULTS = OrderedDict([ + ("example.bool", {"type": "bool", "default": True}), + ("example.int", {"type": "int", "default": 42}), + ("example.string", {"type": "string", "default": "yolo swag"}), + ("example.enum", {"type": "enum", "default": "a", "choices": ["a", "b", "c"]}), +]) + + +def settings_get(key, full=False): + """ + Get an entry value in the settings + + Keyword argument: + key -- Settings key + + """ + settings = _get_settings() + + if key not in settings: + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_key_doesnt_exists', settings_key=key)) + + if full: + return settings[key] + + return settings[key]['value'] + + +def settings_list(): + """ + List all entries of the settings + + """ + return _get_settings() + + +def settings_set(key, value): + """ + Set an entry value in the settings + + Keyword argument: + key -- Settings key + value -- New value + + """ + settings = _get_settings() + + if key not in settings: + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_key_doesnt_exists', settings_key=key)) + + key_type = settings[key]["type"] + + if key_type == "bool": + if not isinstance(value, bool): + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_bad_type_for_setting', setting=key, + received_type=type(value).__name__, expected_type=key_type)) + elif key_type == "int": + if not isinstance(value, int) or isinstance(value, bool): + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_bad_type_for_setting', setting=key, + received_type=type(value).__name__, expected_type=key_type)) + elif key_type == "string": + if not isinstance(value, basestring): + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_bad_type_for_setting', setting=key, + received_type=type(value).__name__, expected_type=key_type)) + elif key_type == "enum": + if value not in settings[key]["choices"]: + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_bad_choice_for_enum', setting=key, + received_type=type(value).__name__, + expected_type=", ".join(settings[key]["choices"]))) + else: + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_unknown_type', setting=key, + unknown_type=key_type)) + + settings[key]["value"] = value + + _save_settings(settings) + + +def settings_reset(key): + """ + Set an entry value to its default one + + Keyword argument: + key -- Settings key + + """ + settings = _get_settings() + + if key not in settings: + raise MoulinetteError(errno.EINVAL, m18n.n( + 'global_settings_key_doesnt_exists', settings_key=key)) + + settings[key]["value"] = settings[key]["default"] + _save_settings(settings) + + +def settings_reset_all(): + """ + Reset all settings to their default value + + Keyword argument: + yes -- Yes I'm sure I want to do that + + """ + settings = _get_settings() + + # For now on, we backup the previous settings in case of but we don't have + # any mecanism to take advantage of those backups. It could be a nice + # addition but we'll see if this is a common need. + # Another solution would be to use etckeeper and integrate those + # modification inside of it and take advantage of its git history + old_settings_backup_path = SETTINGS_PATH_OTHER_LOCATION % datetime.now().strftime("%F_%X") + _save_settings(settings, location=old_settings_backup_path) + + for value in settings.values(): + value["value"] = value["default"] + + _save_settings(settings) + + return { + "old_settings_backup_path": old_settings_backup_path, + "message": m18n.n("global_settings_reset_success", path=old_settings_backup_path) + } + + +def _get_settings(): + settings = {} + + for key, value in DEFAULTS.copy().items(): + settings[key] = value + settings[key]["value"] = value["default"] + settings[key]["description"] = m18n.n("global_settings_setting_%s" % key.replace(".", "_")) + + if not os.path.exists(SETTINGS_PATH): + return settings + + # we have a very strict policy on only allowing settings that we know in + # the OrderedDict DEFAULTS + # For various reason, while reading the local settings we might encounter + # settings that aren't in DEFAULTS, those can come from settings key that + # we have removed, errors or the user trying to modify + # /etc/yunohost/settings.json + # To avoid to simply overwrite them, we store them in + # /etc/yunohost/settings-unknown.json in case of + unknown_settings = {} + unknown_settings_path = SETTINGS_PATH_OTHER_LOCATION % "unknown" + + if os.path.exists(unknown_settings_path): + try: + unknown_settings = json.load(open(unknown_settings_path, "r")) + except Exception as e: + logger.warning("Error while loading unknown settings %s" % e) + + try: + with open(SETTINGS_PATH) as settings_fd: + local_settings = json.load(settings_fd) + + for key, value in local_settings.items(): + if key in settings: + settings[key] = value + settings[key]["description"] = m18n.n("global_settings_setting_%s" % key.replace(".", "_")) + else: + logger.warning(m18n.n('global_settings_unknown_setting_from_settings_file', + setting_key=key)) + unknown_settings[key] = value + except Exception as e: + raise MoulinetteError(errno.EIO, m18n.n('global_settings_cant_open_settings', reason=e), + exc_info=1) + + if unknown_settings: + try: + _save_settings(unknown_settings, location=unknown_settings_path) + except Exception as e: + logger.warning("Failed to save unknown settings (because %s), aborting." % e) + + return settings + + +def _save_settings(settings, location=SETTINGS_PATH): + settings_without_description = {} + for key, value in settings.items(): + settings_without_description[key] = value + if "description" in value: + del settings_without_description[key]["description"] + + try: + result = json.dumps(settings_without_description, indent=4) + except Exception as e: + raise MoulinetteError(errno.EINVAL, + m18n.n('global_settings_cant_serialize_setings', reason=e), + exc_info=1) + + try: + with open(location, "w") as settings_fd: + settings_fd.write(result) + except Exception as e: + raise MoulinetteError(errno.EIO, + m18n.n('global_settings_cant_write_settings', reason=e), + exc_info=1) diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 946eec23c..65c1d3ace 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -105,4 +105,3 @@ def pytest_cmdline_main(config): # Initialize moulinette moulinette.init(logging_config=logging, _from_source=False) - diff --git a/src/yunohost/tests/test_settings.py b/src/yunohost/tests/test_settings.py new file mode 100644 index 000000000..746f5a9d4 --- /dev/null +++ b/src/yunohost/tests/test_settings.py @@ -0,0 +1,166 @@ +import os +import json +import pytest + +from moulinette.core import MoulinetteError + +from yunohost.settings import settings_get, settings_list, _get_settings, \ + settings_set, settings_reset, settings_reset_all, \ + SETTINGS_PATH_OTHER_LOCATION, SETTINGS_PATH + + +def setup_function(function): + os.system("mv /etc/yunohost/settings.json /etc/yunohost/settings.json.saved") + + +def teardown_function(function): + os.system("mv /etc/yunohost/settings.json.saved /etc/yunohost/settings.json") + + +def test_settings_get_bool(): + assert settings_get("example.bool") == True + +def test_settings_get_full_bool(): + assert settings_get("example.bool", True) == {"type": "bool", "value": True, "default": True, "description": "Example boolean option"} + + +def test_settings_get_int(): + assert settings_get("example.int") == 42 + +def test_settings_get_full_int(): + assert settings_get("example.int", True) == {"type": "int", "value": 42, "default": 42, "description": "Example int option"} + + +def test_settings_get_string(): + assert settings_get("example.string") == "yolo swag" + +def test_settings_get_full_string(): + assert settings_get("example.string", True) == {"type": "string", "value": "yolo swag", "default": "yolo swag", "description": "Example string option"} + + +def test_settings_get_enum(): + assert settings_get("example.enum") == "a" + +def test_settings_get_full_enum(): + assert settings_get("example.enum", True) == {"type": "enum", "value": "a", "default": "a", "description": "Example enum option", "choices": ["a", "b", "c"]} + + +def test_settings_get_doesnt_exists(): + with pytest.raises(MoulinetteError): + settings_get("doesnt.exists") + + +def test_settings_list(): + assert settings_list() == _get_settings() + + +def test_settings_set(): + settings_set("example.bool", False) + assert settings_get("example.bool") == False + + +def test_settings_set_int(): + settings_set("example.int", 21) + assert settings_get("example.int") == 21 + + +def test_settings_set_enum(): + settings_set("example.enum", "c") + assert settings_get("example.enum") == "c" + + +def test_settings_set_doesexit(): + with pytest.raises(MoulinetteError): + settings_set("doesnt.exist", True) + + +def test_settings_set_bad_type_bool(): + with pytest.raises(MoulinetteError): + settings_set("example.bool", 42) + with pytest.raises(MoulinetteError): + settings_set("example.bool", "pouet") + + +def test_settings_set_bad_type_int(): + with pytest.raises(MoulinetteError): + settings_set("example.int", True) + with pytest.raises(MoulinetteError): + settings_set("example.int", "pouet") + + +def test_settings_set_bad_type_string(): + with pytest.raises(MoulinetteError): + settings_set("example.string", True) + with pytest.raises(MoulinetteError): + settings_set("example.string", 42) + + +def test_settings_set_bad_value_enum(): + with pytest.raises(MoulinetteError): + settings_set("example.enum", True) + with pytest.raises(MoulinetteError): + settings_set("example.enum", "e") + with pytest.raises(MoulinetteError): + settings_set("example.enum", 42) + with pytest.raises(MoulinetteError): + settings_set("example.enum", "pouet") + + +def test_settings_list_modified(): + settings_set("example.int", 21) + assert settings_list()["example.int"] == {'default': 42, 'description': 'Example int option', 'type': 'int', 'value': 21} + + +def test_reset(): + settings_set("example.int", 21) + assert settings_get("example.int") == 21 + settings_reset("example.int") + assert settings_get("example.int") == settings_get("example.int", True)["default"] + + +def test_settings_reset_doesexit(): + with pytest.raises(MoulinetteError): + settings_reset("doesnt.exist") + + +def test_reset_all(): + settings_before = settings_list() + settings_set("example.bool", False) + settings_set("example.int", 21) + settings_set("example.string", "pif paf pouf") + settings_set("example.enum", "c") + assert settings_before != settings_list() + settings_reset_all() + if settings_before != settings_list(): + for i in settings_before: + assert settings_before[i] == settings_list()[i] + + +def test_reset_all_backup(): + settings_before = settings_list() + settings_set("example.bool", False) + settings_set("example.int", 21) + settings_set("example.string", "pif paf pouf") + settings_set("example.enum", "c") + settings_after_modification = settings_list() + assert settings_before != settings_after_modification + old_settings_backup_path = settings_reset_all()["old_settings_backup_path"] + + for i in settings_after_modification: + del settings_after_modification[i]["description"] + + assert settings_after_modification == json.load(open(old_settings_backup_path, "r")) + + + +def test_unknown_keys(): + unknown_settings_path = SETTINGS_PATH_OTHER_LOCATION % "unknown" + unknown_setting = { + "unkown_key": {"value": 42, "default": 31, "type": "int"}, + } + open(SETTINGS_PATH, "w").write(json.dumps(unknown_setting)) + + # stimulate a write + settings_reset_all() + + assert unknown_setting == json.load(open(unknown_settings_path, "r"))