Tmp work on provision/deprovision apt and system user

This commit is contained in:
Alexandre Aubin 2021-11-13 17:09:30 +01:00
parent 744729713d
commit 7206be0020
2 changed files with 398 additions and 251 deletions

View file

@ -735,6 +735,37 @@ def app_manifest(app):
return manifest
def _confirm_app_install(app, force=False):
# Ignore if there's nothing for confirm (good quality app), if --force is used
# or if request on the API (confirm already implemented on the API side)
if force or Moulinette.interface.type == "api":
return
quality = _app_quality(app)
if quality == "success":
return
# i18n: confirm_app_install_warning
# i18n: confirm_app_install_danger
# i18n: confirm_app_install_thirdparty
if quality in ["danger", "thirdparty"]:
answer = Moulinette.prompt(
m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"),
color="red",
)
if answer != "Yes, I understand":
raise YunohostError("aborting")
else:
answer = Moulinette.prompt(
m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow"
)
if answer.upper() != "Y":
raise YunohostError("aborting")
@is_unit_operation()
def app_install(
operation_logger,
@ -776,37 +807,7 @@ def app_install(
if free_space_in_directory("/") <= 512 * 1000 * 1000:
raise YunohostValidationError("disk_space_not_sufficient_install")
def confirm_install(app):
# Ignore if there's nothing for confirm (good quality app), if --force is used
# or if request on the API (confirm already implemented on the API side)
if force or Moulinette.interface.type == "api":
return
quality = _app_quality(app)
if quality == "success":
return
# i18n: confirm_app_install_warning
# i18n: confirm_app_install_danger
# i18n: confirm_app_install_thirdparty
if quality in ["danger", "thirdparty"]:
answer = Moulinette.prompt(
m18n.n("confirm_app_install_" + quality, answers="Yes, I understand"),
color="red",
)
if answer != "Yes, I understand":
raise YunohostError("aborting")
else:
answer = Moulinette.prompt(
m18n.n("confirm_app_install_" + quality, answers="Y/N"), color="yellow"
)
if answer.upper() != "Y":
raise YunohostError("aborting")
confirm_install(app)
_confirm_app_install(app)
manifest, extracted_app_folder = _extract_app(app)
packaging_format = manifest["packaging_format"]
@ -815,7 +816,6 @@ def app_install(
raise YunohostValidationError("app_id_invalid")
app_id = manifest["id"]
label = label if label else manifest["name"]
# Check requirements
_check_manifest_requirements(manifest, action="install")
@ -829,6 +829,8 @@ def app_install(
else:
app_instance_name = app_id
app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
# Retrieve arguments list for install script
raw_questions = manifest["install"]
questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args)
@ -861,7 +863,6 @@ def app_install(
logger.info(m18n.n("app_start_install", app=app_id))
# Create app directory
app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name)
if os.path.exists(app_setting_path):
shutil.rmtree(app_setting_path)
os.makedirs(app_setting_path)
@ -894,23 +895,31 @@ def app_install(
recursive=True,
)
# Initialize the main permission for the app
# The permission is initialized with no url associated, and with tile disabled
# For web app, the root path of the app will be added as url and the tile
# will be enabled during the app install. C.f. 'app_register_url()' below.
if packaging_format >= 2:
init_main_perm_allowed = ["visitors"] if not args.get("is_public") else ["all_users"]
else:
init_main_perm_allowed = ["all_users"]
resources = AppResourceSet(manifest["resources"], app_instance_name)
resources.check_availability()
resources.provision()
permission_create(
app_instance_name + ".main",
allowed=init_main_perm_allowed,
label=label if label else manifest["name"],
show_tile=False,
protected=False,
)
if packaging_format < 2:
# Initialize the main permission for the app
# The permission is initialized with no url associated, and with tile disabled
# For web app, the root path of the app will be added as url and the tile
# will be enabled during the app install. C.f. 'app_register_url()' below.
permission_create(
app_instance_name + ".main",
allowed=["all_users"],
label=label,
show_tile=False,
protected=False,
)
if packaging_format >= 2:
try:
from yunohost.utils.resources import AppResourceManager
resources = AppResourceManager(app_instance_name, current=app_setting_path, wanted=extracted_app_folder)
resources.apply()
except:
raise
# FIXME : error handling
# Prepare env. var. to pass to script
env_dict = _make_environment_for_app_script(
@ -2519,7 +2528,7 @@ def _make_environment_for_app_script(
# If packaging format v2, load all settings
if manifest["packaging_format"] >= 2:
env_dict["app"] = app
for setting_name, setting_value in _get_app_settings(app):
for setting_name, setting_value in _get_app_settings(app).items():
# Ignore special internal settings like checksum__
# (not a huge deal to load them but idk...)

View file

@ -22,14 +22,47 @@ import os
import copy
from typing import Dict, Any
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.hook import hook_exec
logger = getActionLogger("yunohost.app_resources")
class AppResourceManager(object):
def __init__(self, app: str, manifest: str):
self.app = app
self.resources = {name: AppResourceClassesByType[name](infos, app)
for name, infos in resources_dict.items()}
def apply(self):
def validate_resource_availability(self):
for name, resource in self.resources.items():
resource.validate_availability(context={})
def provision_or_update_resources(self):
for name, resource in self.resources.items():
logger.info("Running provision_or_upgrade for {self.type}")
resource.provision_or_update(context={})
class AppResource(object):
def __init__(self, properties: Dict[str, Any], app_id: str):
def __init__(self, properties: Dict[str, Any], app: str):
self.app_id = app_id
self.app = app
for key, value in self.default_properties.items():
setattr(self, key, value)
@ -37,85 +70,48 @@ class AppResource(object):
for key, value in properties.items():
setattr(self, key, value)
def get_app_settings(self):
from yunohost.app import _get_app_settings
return _get_app_settings(self.app_id)
def get_setting(self, key):
from yunohost.app import app_setting
return app_setting(self.app, key)
def check_availability(self, context: Dict):
def set_setting(self, key, value):
from yunohost.app import app_setting
app_setting(self.app, key, value=value)
def delete_setting(self, key, value):
from yunohost.app import app_setting
app_setting(self.app, key, delete=True)
def validate_availability(self, context: Dict):
pass
def _run_script(self, action, script, env={}, user="root"):
class AppResourceSet:
from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script
def __init__(self, resources_dict: Dict[str, Dict[str, Any]], app_id: str):
tmpdir = _make_tmp_workdir_for_app(app=self.app)
self.set = {name: AppResourceClassesByType[name](infos, app_id)
for name, infos in resources_dict.items()}
env_ = _make_environment_for_app_script(self.app, workdir=tmpdir, action=f"{action}_{self.type}")
env_.update(env)
def check_availability(self):
script_path = f"{tmpdir}/{action}_{self.type}"
script = f"""
source /usr/share/yunohost/helpers
ynh_abort_if_errors
for name, resource in self.set.items():
resource.check_availability(context={})
{script}
"""
write_to_file(script_path, script)
print(env_)
# FIXME : use the hook_exec_with_debug_instructions_stuff
ret, _ = hook_exec(script_path, env=env_)
print(ret)
class AptDependenciesAppResource(AppResource):
"""
is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn)
is_available -> True? idk
update -> update deps on __APP__-ynh-deps
provision -> create/update deps on __APP__-ynh-deps
deprovision -> remove __APP__-ynh-deps (+autoremove?)
deep_clean -> remove any __APP__-ynh-deps for app not in app list
backup -> nothing
restore = provision
"""
type = "apt"
default_properties = {
"packages": [],
"extras": {}
}
def check_availability(self, context):
# ? FIXME
# call helpers idk ...
pass
class SourcesAppResource(AppResource):
"""
is_provisioned -> (if pre_download,) cache exists with appropriate checksum
is_available -> curl HEAD returns 200
update -> none?
provision -> full download + check checksum
deprovision -> remove cache for __APP__ ?
deep_clean -> remove all cache
backup -> nothing
restore -> nothing
"""
type = "sources"
default_properties = {
"main": {"url": "?", "sha256sum": "?", "predownload": True}
}
def check_availability(self, context):
# ? FIXME
# call request.head on the url idk
pass
class RoutesAppResource(AppResource):
class WebpathResource(AppResource):
"""
is_provisioned -> main perm exists
is_available -> perm urls do not conflict
@ -131,101 +127,30 @@ class RoutesAppResource(AppResource):
restore -> handled by the core, should be integrated in there (restore .ldif/yml?)
"""
type = "routes"
type = "webpath"
priority = 10
default_properties = {
"full_domain": False,
"main": {
"url": "/",
"additional_urls": [],
"init_allowed": "__FIXME__",
"show_tile": True,
"protected": False,
"auth_header": True,
"label": "FIXME",
}
}
def check_availability(self, context):
def validate_availability(self, context):
from yunohost.app import _assert_no_conflicting_apps
app_settings = self.get_app_settings()
domain = app_settings["domain"]
path = app_settings["path"] if not self.full_domain else "/"
_assert_no_conflicting_apps(domain, path, ignore_app=self.app_id)
domain = self.get_setting("domain")
path = self.get_setting("path") if not self.full_domain else "/"
_assert_no_conflicting_apps(domain, path, ignore_app=self.app)
def provision_or_update(self, context: Dict):
if context["app_action"] == "install":
pass # FIXME
# Initially, the .main permission is created with no url at all associated
# When the app register/books its web url, we also add the url '/'
# (meaning the root of the app, domain.tld/path/)
# and enable the tile to the SSO, and both of this should match 95% of apps
# For more specific cases, the app is free to change / add urls or disable
# the tile using the permission helpers.
#permission_create(
# self.app_id + ".main",
# allowed=["all_users"],
# label=label,
# show_tile=False,
# protected=False,
#)
#permission_url(app + ".main", url="/", sync_perm=False)
#user_permission_update(app + ".main", show_tile=True, sync_perm=False)
#permission_sync_to_user()
# Nothing to do ? Just setting the domain/path during install
# already provisions it ...
return # FIXME
def deprovision(self, context: Dict):
del context["app_settings"]["domain"]
del context["app_settings"]["path"]
class PortAppResource(AppResource):
"""
is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting)
is_available -> true
update -> true
provision -> find a port not used by any app
deprovision -> delete the port setting
deep_clean -> ?
backup -> nothing (backup port setting)
restore -> nothing (restore port setting)
"""
type = "port"
default_properties = {
"value": 1000,
"type": "internal",
}
def _port_is_used(self, port):
# FIXME : this could be less brutal than two os.system ...
cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port
# This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
cmd2 = f"grep -q \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml"
return os.system(cmd1) == 0 and os.system(cmd2) == 0
def provision_or_update(self, context: str):
# Don't do anything if port already defined ?
if context["app_settings"].get("port"):
return
port = self.value
while self._port_is_used(port):
port += 1
context["app_settings"]["port"] = port
def deprovision(self, context: Dict):
raise NotImplementedError()
self.delete_setting("domain")
self.delete_setting("path")
class SystemuserAppResource(AppResource):
@ -233,8 +158,8 @@ class SystemuserAppResource(AppResource):
is_provisioned -> user __APP__ exists
is_available -> user and group __APP__ doesn't exists
update -> update values for home / shell / groups
provision -> create user
update -> update values for home / shell / groups
deprovision -> delete user
@ -245,26 +170,69 @@ class SystemuserAppResource(AppResource):
"""
type = "system_user"
priority = 20
default_properties = {
"username": "__APP__",
"home_dir": "__INSTALL_DIR__",
"use_shell": False,
"groups": []
"allow_ssh": []
"allow_sftp": []
}
def check_availability(self, context):
if os.system(f"getent passwd {self.username} &>/dev/null") != 0:
raise YunohostValidationError(f"User {self.username} already exists")
if os.system(f"getent group {self.username} &>/dev/null") != 0:
raise YunohostValidationError(f"Group {self.username} already exists")
def validate_availability(self, context):
pass
# FIXME : do we care if user already exists ? shouldnt we assume that user $app corresponds to the app ...?
# FIXME : but maybe we should at least check that no corresponding yunohost user exists
#if os.system(f"getent passwd {self.username} &>/dev/null") != 0:
# raise YunohostValidationError(f"User {self.username} already exists")
#if os.system(f"getent group {self.username} &>/dev/null") != 0:
# raise YunohostValidationError(f"Group {self.username} already exists")
def provision_or_update(self, context: Dict):
if os.system(f"getent passwd {self.app} &>/dev/null") != 0:
cmd = f"useradd --system --user-group {self.app}"
os.system(cmd)
if os.system(f"getent passwd {self.app} &>/dev/null") == 0:
raise YunohostError(f"Failed to create system user for {self.app}")
groups = []
if self.allow_ssh:
groups.append("ssh.app")
if self.allow_sftp:
groups.append("sftp.app")
groups =
cmd = f"usermod -a -G {groups} {self.app}"
# FIXME : handle case where group gets removed
os.system(cmd)
# useradd $user_home_dir --system --user-group $username $shell || ynh_die --message="Unable to create $username system account"
# for group in $groups; do
# usermod -a -G "$group" "$username"
# done
# | arg: -g, --groups - Add the user to system groups. Typically meant to add the user to the ssh.app / sftp.app group (e.g. for borgserver, my_webapp)
def provision_or_update(self, context: str):
raise NotImplementedError()
def deprovision(self, context: Dict):
raise NotImplementedError()
self._run_script("deprovision",
f'ynh_system_user_delete "{self.username}"')
# # Check if the user exists on the system
#if os.system(f"getent passwd {self.username} &>/dev/null") != 0:
# if ynh_system_user_exists "$username"; then
# deluser $username
# fi
# # Check if the group exists on the system
#if os.system(f"getent group {self.username} &>/dev/null") != 0:
# if ynh_system_group_exists "$username"; then
# delgroup $username
# fi
#
class InstalldirAppResource(AppResource):
"""
@ -283,32 +251,54 @@ class InstalldirAppResource(AppResource):
"""
type = "install_dir"
priority = 30
default_properties = {
"dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk...
"alias": "final_path",
# FIXME : add something about perms ?
"alias": None,
"owner": "__APP__:rx",
"group": "__APP__:rx",
}
# FIXME: change default dir to /opt/stuff if app ain't a webapp ...
# FIXME: what do in a scenario where the location changed
def check_availability(self, context):
if os.path.exists(self.dir):
raise YunohostValidationError(f"Folder {self.dir} already exists")
def validate_availability(self, context):
pass
def provision_or_update(self, context: Dict):
if context["app_action"] in ["install", "restore"]:
if os.path.exists(self.dir):
raise YunohostValidationError(f"Path {self.dir} already exists")
current_install_dir = self.get_setting("install_dir")
if "installdir" not in context["app_settings"]:
context["app_settings"]["installdir"] = self.dir
context["app_settings"][self.alias] = context["app_settings"]["installdir"]
# If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it
# FIXME : is this the right thing to do ?
if not current_install_dir and os.path.isdir(self.dir):
rm(self.dir, recursive=True)
if not os.path.isdir(self.dir):
# Handle case where install location changed, in which case we shall move the existing install dir
if current_install_dir and os.path.isdir(current_install_dir):
shutil.move(current_install_dir, self.dir)
else:
mkdir(self.dir)
owner, owner_perm = self.owner.split(":")
group, group_perm = self.group.split(":")
owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0"
chmod(self.dir, oct(int(perm_octal)))
chown(self.dir, owner, group)
self.set_setting("install_dir", self.dir)
if self.alias:
self.set_setting(self.alias, self.dir)
def deprovision(self, context: Dict):
# FIXME: should it rm the directory during remove/deprovision ?
pass
# FIXME : check that self.dir has a sensible value to prevent catastrophes
if os.path.isdir(self.dir):
rm(self.dir, recursive=True)
class DatadirAppResource(AppResource):
@ -328,56 +318,204 @@ class DatadirAppResource(AppResource):
"""
type = "data_dir"
priority = 40
default_properties = {
"dir": "/home/yunohost.app/__APP__", # FIXME or choose to move this elsewhere nowadays idk...
"owner": "__APP__:rx",
"group": "__APP__:rx",
}
def check_availability(self, context):
if os.path.exists(self.dir):
raise YunohostValidationError(f"Folder {self.dir} already exists")
def validate_availability(self, context):
pass
# Nothing to do ? If datadir already exists then it may be legit data
# from a previous install
def provision_or_update(self, context: Dict):
if "datadir" not in context["app_settings"]:
context["app_settings"]["datadir"] = self.dir
current_data_dir = self.get_setting("data_dir")
if not os.path.isdir(self.dir):
# Handle case where install location changed, in which case we shall move the existing install dir
if current_data_dir and os.path.isdir(current_data_dir):
shutil.move(current_data_dir, self.dir)
else:
mkdir(self.dir)
owner, owner_perm = self.owner.split(":")
group, group_perm = self.group.split(":")
owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
perm_octal = str(owner_perm_octal) + str(group_perm_octal) + "0"
chmod(self.dir, oct(int(perm_octal)))
chown(self.dir, owner, group)
self.set_setting("data_dir", self.dir)
def deprovision(self, context: Dict):
# FIXME: should it rm the directory during remove/deprovision ?
# FIXME: This should rm the datadir only if purge is enabled
pass
#if os.path.isdir(self.dir):
# rm(self.dir, recursive=True)
class DBAppResource(AppResource):
#
#class SourcesAppResource(AppResource):
# """
# is_provisioned -> (if pre_download,) cache exists with appropriate checksum
# is_available -> curl HEAD returns 200
#
# update -> none?
# provision -> full download + check checksum
#
# deprovision -> remove cache for __APP__ ?
#
# deep_clean -> remove all cache
#
# backup -> nothing
# restore -> nothing
# """
#
# type = "sources"
#
# default_properties = {
# "main": {"url": "?", "sha256sum": "?", "predownload": True}
# }
#
# def validate_availability(self, context):
# # ? FIXME
# # call request.head on the url idk
# pass
#
# def provision_or_update(self, context: Dict):
# # FIXME
# return
#
class AptDependenciesAppResource(AppResource):
"""
is_provisioned -> setting db_user, db_name, db_pwd exists
is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it)
is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn)
is_available -> True? idk
provision -> setup the db + init the setting
update -> ??
update -> update deps on __APP__-ynh-deps
provision -> create/update deps on __APP__-ynh-deps
deprovision -> delete the db
deprovision -> remove __APP__-ynh-deps (+autoremove?)
deep_clean -> ... idk look into any db name that would not be related to any app ...
deep_clean -> remove any __APP__-ynh-deps for app not in app list
backup -> dump db
restore -> setup + inject db dump
backup -> nothing
restore = provision
"""
type = "db"
type = "apt"
priority = 50
default_properties = {
"type": "mysql"
"packages": [],
"extras": {}
}
def check_availability(self, context):
# FIXME : checking availability sort of imply that mysql / postgresql is installed
# or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource)
def validate_availability(self, context):
# ? FIXME
# call helpers idk ...
pass
def provision_or_update(self, context: str):
raise NotImplementedError()
def provision_or_update(self, context: Dict):
# FIXME : implement 'extras' management
self._run_script("provision_or_update",
"ynh_install_app_dependencies $apt_dependencies",
{"apt_dependencies": self.packages})
def deprovision(self, context: Dict):
raise NotImplementedError()
self._run_script("deprovision",
"ynh_remove_app_dependencies")
class PortAppResource(AppResource):
"""
is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting)
is_available -> true
update -> true
provision -> find a port not used by any app
deprovision -> delete the port setting
deep_clean -> ?
backup -> nothing (backup port setting)
restore -> nothing (restore port setting)
"""
type = "port"
priority = 70
default_properties = {
"default": 1000,
"type": "internal", # FIXME : implement logic for exposed port (allow/disallow in firewall ?)
}
def _port_is_used(self, port):
# FIXME : this could be less brutal than two os.system ...
cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port
# This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
cmd2 = f"grep --quiet \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml"
return os.system(cmd1) == 0 and os.system(cmd2) == 0
def provision_or_update(self, context: str):
# Don't do anything if port already defined ?
if self.get_setting("port"):
return
port = self.default
while self._port_is_used(port):
port += 1
self.set_setting("port", port)
def deprovision(self, context: Dict):
self.delete_setting("port")
#class DBAppResource(AppResource):
# """
# is_provisioned -> setting db_user, db_name, db_pwd exists
# is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it)
#
# provision -> setup the db + init the setting
# update -> ??
#
# deprovision -> delete the db
#
# deep_clean -> ... idk look into any db name that would not be related to any app ...
#
# backup -> dump db
# restore -> setup + inject db dump
# """
#
# type = "db"
#
# default_properties = {
# "type": "mysql"
# }
#
# def validate_availability(self, context):
# # FIXME : checking availability sort of imply that mysql / postgresql is installed
# # or we gotta make sure mariadb-server or postgresql is gonna be installed (apt resource)
# pass
#
# def provision_or_update(self, context: str):
# raise NotImplementedError()
#
# def deprovision(self, context: Dict):
# raise NotImplementedError()
#
AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()}