# -*- coding: utf-8 -*-
""" License
Copyright (C) 2021 YUNOHOST.ORG
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, see http://www.gnu.org/licenses
"""
import os
import copy
import shutil
import random
from typing import Dict, Any
from moulinette.utils.process import check_output
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
from moulinette.utils.filesystem import (
rm,
)
from yunohost.utils.error import YunohostError
from yunohost.hook import hook_exec
logger = getActionLogger("yunohost.app_resources")
class AppResourceManager:
def __init__(self, app: str, current: Dict, wanted: Dict):
self.app = app
self.current = current
self.wanted = wanted
def apply(self, rollback_if_failure, **context):
todos = list(self.compute_todos())
completed = []
rollback = False
exception = None
for todo, name, old, new in todos:
try:
if todo == "deprovision":
# FIXME : i18n, better info strings
logger.info(f"Deprovisionning {name} ...")
old.deprovision(context=context)
elif todo == "provision":
logger.info(f"Provisionning {name} ...")
new.provision_or_update(context=context)
elif todo == "update":
logger.info(f"Updating {name} ...")
new.provision_or_update(context=context)
except Exception as e:
exception = e
# FIXME: better error handling ? display stacktrace ?
logger.warning(f"Failed to {todo} for {name} : {e}")
if rollback_if_failure:
rollback = True
completed.append((todo, name, old, new))
break
else:
pass
else:
completed.append((todo, name, old, new))
if rollback:
for todo, name, old, new in completed:
try:
# (NB. here we want to undo the todo)
if todo == "deprovision":
# FIXME : i18n, better info strings
logger.info(f"Reprovisionning {name} ...")
old.provision_or_update(context=context)
elif todo == "provision":
logger.info(f"Deprovisionning {name} ...")
new.deprovision(context=context)
elif todo == "update":
logger.info(f"Reverting {name} ...")
old.provision_or_update(context=context)
except Exception as e:
# FIXME: better error handling ? display stacktrace ?
logger.error(f"Failed to rollback {name} : {e}")
if exception:
raise exception
def compute_todos(self):
for name, infos in reversed(self.current["resources"].items()):
if name not in self.wanted["resources"].keys():
resource = AppResourceClassesByType[name](infos, self.app, self)
yield ("deprovision", name, resource, None)
for name, infos in self.wanted["resources"].items():
wanted_resource = AppResourceClassesByType[name](infos, self.app, self)
if name not in self.current["resources"].keys():
yield ("provision", name, None, wanted_resource)
else:
infos_ = self.current["resources"][name]
current_resource = AppResourceClassesByType[name](infos_, self.app, self)
yield ("update", name, current_resource, wanted_resource)
class AppResource:
def __init__(self, properties: Dict[str, Any], app: str, manager=None):
self.app = app
self.manager = manager
for key, value in self.default_properties.items():
if isinstance(value, str):
value = value.replace("__APP__", self.app)
setattr(self, key, value)
for key, value in properties.items():
if isinstance(value, str):
value = value.replace("__APP__", self.app)
setattr(self, key, value)
def get_setting(self, key):
from yunohost.app import app_setting
return app_setting(self.app, key)
def set_setting(self, key, value):
from yunohost.app import app_setting
app_setting(self.app, key, value=value)
def delete_setting(self, key):
from yunohost.app import app_setting
app_setting(self.app, key, delete=True)
def _run_script(self, action, script, env={}, user="root"):
from yunohost.app import _make_tmp_workdir_for_app, _make_environment_for_app_script
tmpdir = _make_tmp_workdir_for_app(app=self.app)
env_ = _make_environment_for_app_script(self.app, workdir=tmpdir, action=f"{action}_{self.type}")
env_.update(env)
script_path = f"{tmpdir}/{action}_{self.type}"
script = f"""
source /usr/share/yunohost/helpers
ynh_abort_if_errors
{script}
"""
write_to_file(script_path, script)
#print(env_)
# FIXME : use the hook_exec_with_debug_instructions_stuff
ret, _ = hook_exec(script_path, env=env_)
#print(ret)
class PermissionsResource(AppResource):
"""
is_provisioned -> main perm exists
is_available -> perm urls do not conflict
update -> refresh/update values for url/additional_urls/show_tile/auth/protected/... create new perms / delete any perm not listed
provision -> same as update?
deprovision -> delete permissions
deep_clean -> delete permissions for any __APP__.foobar where app not in app list...
backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?)
restore -> handled by the core, should be integrated in there (restore .ldif/yml?)
"""
type = "permissions"
priority = 10
default_properties = {
}
default_perm_properties = {
"url": None,
"additional_urls": [],
"auth_header": True,
"allowed": None,
"show_tile": None, # To be automagically set to True by default if an url is defined and show_tile not provided
"protected": False,
}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
# FIXME : if url != None, we should check that there's indeed a domain/path defined ? ie that app is a webapp
for perm, infos in properties.items():
properties[perm] = copy.copy(self.default_perm_properties)
properties[perm].update(infos)
if properties[perm]["show_tile"] is None:
properties[perm]["show_tile"] = bool(properties[perm]["url"])
if isinstance(properties["main"]["url"], str) and properties["main"]["url"] != "/":
raise YunohostError("URL for the 'main' permission should be '/' for webapps (or undefined/None for non-webapps). Note that / refers to the install url of the app")
super().__init__({"permissions": properties}, *args, **kwargs)
def provision_or_update(self, context: Dict={}):
from yunohost.permission import (
permission_create,
#permission_url,
permission_delete,
user_permission_list,
user_permission_update,
permission_sync_to_user,
)
# Delete legacy is_public setting if not already done
self.delete_setting("is_public")
existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"]
for perm in existing_perms:
if perm.split(".") not in self.permissions.keys():
permission_delete(perm, force=True, sync_perm=False)
for perm, infos in self.permissions.items():
if f"{self.app}.{perm}" not in existing_perms:
# Use the 'allowed' key from the manifest,
# or use the 'init_{perm}_permission' from the install questions
# which is temporarily saved as a setting as an ugly hack to pass the info to this piece of code...
init_allowed = infos["allowed"] or self.get_setting(f"init_{perm}_permission") or []
permission_create(
f"{self.app}.{perm}",
allowed=init_allowed,
# This is why the ugly hack with self.manager exists >_>
label=self.manager.wanted["name"] if perm == "main" else perm,
url=infos["url"],
additional_urls=infos["additional_urls"],
auth_header=infos["auth_header"],
sync_perm=False,
)
self.delete_setting(f"init_{perm}_permission")
user_permission_update(
f"{self.app}.{perm}",
show_tile=infos["show_tile"],
protected=infos["protected"],
sync_perm=False
)
else:
pass
# FIXME : current implementation of permission_url is hell for
# easy declarativeness of additional_urls >_> ...
#permission_url(f"{self.app}.{perm}", url=infos["url"], auth_header=infos["auth_header"], sync_perm=False)
permission_sync_to_user()
def deprovision(self, context: Dict={}):
from yunohost.permission import (
permission_delete,
user_permission_list,
permission_sync_to_user,
)
existing_perms = user_permission_list(short=True, apps=[self.app])["permissions"]
for perm in existing_perms:
permission_delete(perm, force=True, sync_perm=False)
permission_sync_to_user()
class SystemuserAppResource(AppResource):
"""
is_provisioned -> user __APP__ exists
is_available -> user and group __APP__ doesn't exists
provision -> create user
update -> update values for home / shell / groups
deprovision -> delete user
deep_clean -> uuuuh ? delete any user that could correspond to an app x_x ?
backup -> nothing
restore -> provision
"""
type = "system_user"
priority = 20
default_properties = {
"allow_ssh": False,
"allow_sftp": False
}
def provision_or_update(self, context: Dict={}):
# FIXME : validate that no yunohost user exists with that name?
# and/or that no system user exists during install ?
if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
# FIXME: improve error handling ?
cmd = f"useradd --system --user-group {self.app}"
os.system(cmd)
if not check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
raise YunohostError(f"Failed to create system user for {self.app}", raw_msg=True)
groups = set(check_output(f"groups {self.app}").strip().split()[2:])
if self.allow_ssh:
groups.add("ssh.app")
if self.allow_sftp:
groups.add("sftp.app")
os.system(f"usermod -G {','.join(groups)} {self.app}")
def deprovision(self, context: Dict={}):
if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
os.system(f"deluser {self.app} >/dev/null")
if check_output(f"getent passwd {self.app} &>/dev/null || true").strip():
raise YunohostError(f"Failed to delete system user for {self.app}")
if check_output(f"getent group {self.app} &>/dev/null || true").strip():
os.system(f"delgroup {self.app} >/dev/null")
if check_output(f"getent group {self.app} &>/dev/null || true").strip():
raise YunohostError(f"Failed to delete system user for {self.app}")
# FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands...
# # Check if the user exists on the system
#if os.system(f"getent passwd {self.username} &>/dev/null") != 0:
# if ynh_system_user_exists "$username"; then
# deluser $username
# fi
# # Check if the group exists on the system
#if os.system(f"getent group {self.username} &>/dev/null") != 0:
# if ynh_system_group_exists "$username"; then
# delgroup $username
# fi
#
class InstalldirAppResource(AppResource):
"""
is_provisioned -> setting install_dir exists + /dir/ exists
is_available -> /dir/ doesn't exists
provision -> create setting + create dir
update -> update perms ?
deprovision -> delete dir + delete setting
deep_clean -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ?
backup -> cp install dir
restore -> cp install dir
"""
type = "install_dir"
priority = 30
default_properties = {
"dir": "/var/www/__APP__", # FIXME or choose to move this elsewhere nowadays idk...
"owner": "__APP__:rx",
"group": "__APP__:rx",
}
# FIXME: change default dir to /opt/stuff if app ain't a webapp ...
def provision_or_update(self, context: Dict={}):
current_install_dir = self.get_setting("install_dir") or self.get_setting("final_path")
# If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it
# FIXME : is this the right thing to do ?
if not current_install_dir and os.path.isdir(self.dir):
rm(self.dir, recursive=True)
if not os.path.isdir(self.dir):
# Handle case where install location changed, in which case we shall move the existing install dir
# FIXME: confirm that's what we wanna do
if current_install_dir and os.path.isdir(current_install_dir):
shutil.move(current_install_dir, self.dir)
else:
mkdir(self.dir)
owner, owner_perm = self.owner.split(":")
group, group_perm = self.group.split(":")
owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal
chmod(self.dir, perm_octal)
chown(self.dir, owner, group)
# FIXME: shall we apply permissions recursively ?
self.set_setting("install_dir", self.dir)
self.delete_setting("final_path") # Legacy
def deprovision(self, context: Dict={}):
# FIXME : check that self.dir has a sensible value to prevent catastrophes
if os.path.isdir(self.dir):
rm(self.dir, recursive=True)
# FIXME : in fact we should delete settings to be consistent
class DatadirAppResource(AppResource):
"""
is_provisioned -> setting data_dir exists + /dir/ exists
is_available -> /dir/ doesn't exists
provision -> create setting + create dir
update -> update perms ?
deprovision -> (only if purge enabled...) delete dir + delete setting
deep_clean -> zblerg idk nothing
backup -> cp data dir ? (if not backup_core_only)
restore -> cp data dir ? (if in backup)
"""
type = "data_dir"
priority = 40
default_properties = {
"dir": "/home/yunohost.app/__APP__", # FIXME or choose to move this elsewhere nowadays idk...
"owner": "__APP__:rx",
"group": "__APP__:rx",
}
def provision_or_update(self, context: Dict={}):
current_data_dir = self.get_setting("data_dir")
if not os.path.isdir(self.dir):
# Handle case where install location changed, in which case we shall move the existing install dir
if current_data_dir and os.path.isdir(current_data_dir):
shutil.move(current_data_dir, self.dir)
else:
mkdir(self.dir)
owner, owner_perm = self.owner.split(":")
group, group_perm = self.group.split(":")
owner_perm_octal = (4 if "r" in owner_perm else 0) + (2 if "w" in owner_perm else 0) + (1 if "x" in owner_perm else 0)
group_perm_octal = (4 if "r" in group_perm else 0) + (2 if "w" in group_perm else 0) + (1 if "x" in group_perm else 0)
perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal
chmod(self.dir, perm_octal)
chown(self.dir, owner, group)
self.set_setting("data_dir", self.dir)
def deprovision(self, context: Dict={}):
# FIXME: This should rm the datadir only if purge is enabled
pass
#if os.path.isdir(self.dir):
# rm(self.dir, recursive=True)
# FIXME : in fact we should delete settings to be consistent
#
#class SourcesAppResource(AppResource):
# """
# is_provisioned -> (if pre_download,) cache exists with appropriate checksum
# is_available -> curl HEAD returns 200
#
# update -> none?
# provision -> full download + check checksum
#
# deprovision -> remove cache for __APP__ ?
#
# deep_clean -> remove all cache
#
# backup -> nothing
# restore -> nothing
# """
#
# type = "sources"
#
# default_properties = {
# "main": {"url": "?", "sha256sum": "?", "predownload": True}
# }
#
# def provision_or_update(self, context: Dict={}):
# # FIXME
# return
#
class AptDependenciesAppResource(AppResource):
"""
is_provisioned -> package __APP__-ynh-deps exists (ideally should check the Depends: but hmgn)
is_available -> True? idk
update -> update deps on __APP__-ynh-deps
provision -> create/update deps on __APP__-ynh-deps
deprovision -> remove __APP__-ynh-deps (+autoremove?)
deep_clean -> remove any __APP__-ynh-deps for app not in app list
backup -> nothing
restore = provision
"""
type = "apt"
priority = 50
default_properties = {
"packages": [],
"extras": {}
}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
for key, values in properties.get("extras", {}).items():
if not all(isinstance(values.get(k), str) for k in ["repo", "key", "packages"]):
raise YunohostError("In apt resource in the manifest: 'extras' repo should have the keys 'repo', 'key' and 'packages' defined and be strings")
super().__init__(properties, *args, **kwargs)
def provision_or_update(self, context: Dict={}):
script = [f"ynh_install_app_dependencies {self.packages}"]
for repo, values in self.extras.items():
script += [f"ynh_install_extra_app_dependencies --repo='{values['repo']}' --key='{values['key']}' --package='{values['packages']}'"]
self._run_script("provision_or_update", '\n'.join(script))
def deprovision(self, context: Dict={}):
self._run_script("deprovision", "ynh_remove_app_dependencies")
class PortsResource(AppResource):
"""
is_provisioned -> port setting exists and is not the port used by another app (ie not in another app setting)
is_available -> true
update -> true
provision -> find a port not used by any app
deprovision -> delete the port setting
deep_clean -> ?
backup -> nothing (backup port setting)
restore -> nothing (restore port setting)
"""
type = "ports"
priority = 70
default_properties = {
}
default_port_properties = {
"default": None,
"exposed": False, # or True(="Both"), "TCP", "UDP" # FIXME : implement logic for exposed port (allow/disallow in firewall ?)
"fixed": False, # FIXME: implement logic. Corresponding to wether or not the port is "fixed" or any random port is ok
}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
if "main" not in properties:
properties["main"] = {}
for port, infos in properties.items():
properties[port] = copy.copy(self.default_port_properties)
properties[port].update(infos)
if properties[port]["default"] is None:
properties[port]["default"] = random.randint(10000, 60000)
super().__init__({"ports": properties}, *args, **kwargs)
def _port_is_used(self, port):
# FIXME : this could be less brutal than two os.system ...
cmd1 = "ss --numeric --listening --tcp --udp | awk '{print$5}' | grep --quiet --extended-regexp ':%s$'" % port
# This second command is mean to cover (most) case where an app is using a port yet ain't currently using it for some reason (typically service ain't up)
cmd2 = f"grep --quiet \"port: '{port}'\" /etc/yunohost/apps/*/settings.yml"
return os.system(cmd1) == 0 and os.system(cmd2) == 0
def provision_or_update(self, context: Dict={}):
for name, infos in self.ports.items():
setting_name = f"port_{name}" if name != "main" else "port"
port_value = self.get_setting(setting_name)
if not port_value and name != "main":
# Automigrate from legacy setting foobar_port (instead of port_foobar)
legacy_setting_name = "{name}_port"
port_value = self.get_setting(legacy_setting_name)
self.set_setting(setting_name, port_value)
self.delete_setting(legacy_setting_name)
continue
if not port_value:
port_value = self.infos["default"]
while self._port_is_used(port_value):
port_value += 1
self.set_setting(setting_name, port_value)
def deprovision(self, context: Dict={}):
for name, infos in self.ports.items():
setting_name = f"port_{name}" if name != "main" else "port"
self.delete_setting(setting_name)
class DatabaseAppResource(AppResource):
"""
is_provisioned -> setting db_user, db_name, db_pwd exists
is_available -> db doesn't already exists ( ... also gotta make sure that mysql / postgresql is indeed installed ... or will be after apt provisions it)
provision -> setup the db + init the setting
update -> ??
deprovision -> delete the db
deep_clean -> ... idk look into any db name that would not be related to any app ...
backup -> dump db
restore -> setup + inject db dump
"""
type = "database"
default_properties = {
"type": None,
}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
if "type" not in properties or properties["type"] not in ["mysql", "postgresql"]:
raise YunohostError("Specifying the type of db ('mysql' or 'postgresql') is mandatory for db resources")
super().__init__(properties, *args, **kwargs)
def db_exists(self, db_name):
if self.type == "mysql":
return os.system(f"mysqlshow '{db_name}' >/dev/null 2>/dev/null") == 0
elif self.type == "postgresql":
return os.system(f"sudo --login --user=postgres psql -c '' '{db_name}' >/dev/null 2>/dev/null") == 0
else:
return False
def provision_or_update(self, context: Dict={}):
# This is equivalent to ynh_sanitize_dbid
db_name = self.app.replace('-', '_').replace('.', '_')
db_user = db_name
self.set_setting("db_name", db_name)
self.set_setting("db_user", db_user)
db_pwd = None
if self.get_setting("db_pwd"):
db_pwd = self.get_setting("db_pwd")
else:
# Legacy setting migration
legacypasswordsetting = "psqlpwd" if self.type == "postgresql" else "mysqlpwd"
if self.get_setting(legacypasswordsetting):
db_pwd = self.get_setting(legacypasswordsetting)
self.delete_setting(legacypasswordsetting)
self.set_setting("db_pwd", db_pwd)
if not db_pwd:
from moulinette.utils.text import random_ascii
db_pwd = random_ascii(24)
self.set_setting("db_pwd", db_pwd)
if not self.db_exists(db_name):
if self.type == "mysql":
self._run_script("provision", f"ynh_mysql_create_db '{db_name}' '{db_user}' '{db_pwd}'")
elif self.type == "postgresql":
self._run_script("provision", f"ynh_psql_create_user '{db_user}' '{db_pwd}'; ynh_psql_create_db '{db_name}' '{db_user}'")
def deprovision(self, context: Dict={}):
db_name = self.app.replace('-', '_').replace('.', '_')
db_user = db_name
if self.type == "mysql":
self._run_script("deprovision", f"ynh_mysql_remove_db '{db_name}' '{db_user}'")
elif self.type == "postgresql":
self._run_script("deprovision", f"ynh_psql_remove_db '{db_name}' '{db_user}'")
self.delete_setting("db_name")
self.delete_setting("db_user")
self.delete_setting("db_pwd")
AppResourceClassesByType = {c.type: c for c in AppResource.__subclasses__()}