Cleanup / refactor a bunch of utils into utils/system.py

This commit is contained in:
Alexandre Aubin 2021-11-03 20:38:51 +01:00
parent 6a437c0b4f
commit 810534c661
11 changed files with 146 additions and 224 deletions

View file

@ -43,7 +43,7 @@ _global:
help: Display YunoHost packages versions help: Display YunoHost packages versions
action: callback action: callback
callback: callback:
method: yunohost.utils.packages.ynh_packages_version method: yunohost.utils.system.ynh_packages_version
return: true return: true
############################# #############################

View file

@ -7,7 +7,11 @@ import subprocess
from moulinette.utils.process import check_output from moulinette.utils.process import check_output
from moulinette.utils.filesystem import read_file, read_json, write_to_json from moulinette.utils.filesystem import read_file, read_json, write_to_json
from yunohost.diagnosis import Diagnoser from yunohost.diagnosis import Diagnoser
from yunohost.utils.packages import ynh_packages_version from yunohost.utils.system import (
ynh_packages_version,
system_virt,
system_arch,
)
class BaseSystemDiagnoser(Diagnoser): class BaseSystemDiagnoser(Diagnoser):
@ -18,15 +22,12 @@ class BaseSystemDiagnoser(Diagnoser):
def run(self): def run(self):
# Detect virt technology (if not bare metal) and arch virt = system_virt()
# Gotta have this "|| true" because it systemd-detect-virt return 'none'
# with an error code on bare metal ~.~
virt = check_output("systemd-detect-virt || true", shell=True)
if virt.lower() == "none": if virt.lower() == "none":
virt = "bare-metal" virt = "bare-metal"
# Detect arch # Detect arch
arch = check_output("dpkg --print-architecture") arch = system_arch()
hardware = dict( hardware = dict(
meta={"test": "hardware"}, meta={"test": "hardware"},
status="INFO", status="INFO",

View file

@ -53,7 +53,6 @@ from moulinette.utils.filesystem import (
chmod, chmod,
) )
from yunohost.utils.packages import dpkg_is_broken, get_ynh_package_version
from yunohost.utils.config import ( from yunohost.utils.config import (
ConfigPanel, ConfigPanel,
ask_questions_and_parse_answers, ask_questions_and_parse_answers,
@ -63,7 +62,15 @@ from yunohost.utils.config import (
from yunohost.utils.resources import AppResourceSet from yunohost.utils.resources import AppResourceSet
from yunohost.utils.i18n import _value_for_locale from yunohost.utils.i18n import _value_for_locale
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.filesystem import free_space_in_directory from yunohost.utils.system import (
free_space_in_directory,
dpkg_is_broken,
get_ynh_package_version,
system_arch,
human_to_binary,
binary_to_human,
ram_available,
)
from yunohost.log import is_unit_operation, OperationLogger from yunohost.log import is_unit_operation, OperationLogger
from yunohost.app_catalog import ( # noqa from yunohost.app_catalog import ( # noqa
app_catalog, app_catalog,
@ -179,9 +186,7 @@ def app_info(app, full=False):
ret["supports_backup_restore"] = os.path.exists( ret["supports_backup_restore"] = os.path.exists(
os.path.join(setting_path, "scripts", "backup") os.path.join(setting_path, "scripts", "backup")
) and os.path.exists(os.path.join(setting_path, "scripts", "restore")) ) and os.path.exists(os.path.join(setting_path, "scripts", "restore"))
ret["supports_multi_instance"] = is_true( ret["supports_multi_instance"] = local_manifest.get("integration", {}).get("multi_instance", False)
local_manifest.get("integration", {}).get("multi_instance", False)
)
ret["supports_config_panel"] = os.path.exists( ret["supports_config_panel"] = os.path.exists(
os.path.join(setting_path, "config_panel.toml") os.path.join(setting_path, "config_panel.toml")
) )
@ -1993,7 +1998,7 @@ def _convert_v1_manifest_to_v2(manifest):
manifest["integration"] = { manifest["integration"] = {
"yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""), "yunohost": manifest.get("requirements", {}).get("yunohost", "").replace(">", "").replace("=", "").replace(" ", ""),
"architectures": "all", "architectures": "all",
"multi_instance": is_true(manifest.get("multi_instance", False)), "multi_instance": manifest.get("multi_instance", False),
"ldap": "?", "ldap": "?",
"sso": "?", "sso": "?",
"disk": "50M", "disk": "50M",
@ -2314,12 +2319,12 @@ def _check_manifest_requirements(manifest: Dict, action: str):
app_id = manifest["id"] app_id = manifest["id"]
logger.debug(m18n.n("app_requirements_checking", app=app)) logger.debug(m18n.n("app_requirements_checking", app=app_id))
# Yunohost version requirement # Yunohost version requirement
yunohost_requirement = version.parse(manifest["integration"]["yunohost"] or "4.3") yunohost_requirement = version.parse(manifest["integration"]["yunohost"] or "4.3")
yunohost_installed_version = get_ynh_package_version("yunohost")["version"] yunohost_installed_version = version.parse(get_ynh_package_version("yunohost")["version"])
if yunohost_requirement > yunohost_installed_version: if yunohost_requirement > yunohost_installed_version:
# FIXME : i18n # FIXME : i18n
raise YunohostValidationError(f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}") raise YunohostValidationError(f"This app requires Yunohost >= {yunohost_requirement} but current installed version is {yunohost_installed_version}")
@ -2327,7 +2332,7 @@ def _check_manifest_requirements(manifest: Dict, action: str):
# Architectures # Architectures
arch_requirement = manifest["integration"]["architectures"] arch_requirement = manifest["integration"]["architectures"]
if arch_requirement != "all": if arch_requirement != "all":
arch = check_output("dpkg --print-architecture") arch = system_arch()
if arch not in arch_requirement: if arch not in arch_requirement:
# FIXME: i18n # FIXME: i18n
raise YunohostValidationError(f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}") raise YunohostValidationError(f"This app can only be installed on architectures {', '.join(arch_requirement)} but your server architecture is {arch}")
@ -2343,23 +2348,23 @@ def _check_manifest_requirements(manifest: Dict, action: str):
if action == "install": if action == "install":
disk_requirement = manifest["integration"]["disk"] disk_requirement = manifest["integration"]["disk"]
if free_space_in_directory("/") <= human_to_binary[disk_requirement] \ if free_space_in_directory("/") <= human_to_binary(disk_requirement) \
or free_space_in_directory("/var") <= human_to_binary[disk_requirement]: or free_space_in_directory("/var") <= human_to_binary(disk_requirement):
# FIXME : i18m # FIXME : i18m
raise YunohostValidationError("This app requires {disk_requirement} free space.") raise YunohostValidationError(f"This app requires {disk_requirement} free space.")
# Ram for build # Ram for build
import psutil
ram_build_requirement = manifest["integration"]["ram"]["build"] ram_build_requirement = manifest["integration"]["ram"]["build"]
ram_include_swap = manifest["integration"]["ram"]["include_swap"] ram_include_swap = manifest["integration"]["ram"]["include_swap"]
ram_available = psutil.virtual_memory().available ram, swap = ram_available()
if ram_include_swap: if ram_include_swap:
ram_available += psutil.swap_memory().available ram += swap
if ram_available < human_to_binary(ram_build_requirement): if ram < human_to_binary(ram_build_requirement):
# FIXME : i18n # FIXME : i18n
raise YunohostValidationError("This app requires {ram_build_requirement} RAM available to install/upgrade") ram_human = binary_to_human(ram)
raise YunohostValidationError(f"This app requires {ram_build_requirement} RAM to install/upgrade but only {ram_human} is available right now.")
def _guess_webapp_path_requirement(app_folder: str) -> str: def _guess_webapp_path_requirement(app_folder: str) -> str:
@ -2499,7 +2504,7 @@ def _make_environment_for_app_script(
"YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb),
"YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"), "YNH_APP_MANIFEST_VERSION": manifest.get("version", "?"),
"YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]), "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]),
"YNH_ARCH": check_output("dpkg --print-architecture"), "YNH_ARCH": system_arch(),
} }
if workdir: if workdir:
@ -2607,26 +2612,6 @@ def _make_tmp_workdir_for_app(app=None):
return tmpdir return tmpdir
def is_true(arg):
"""
Convert a string into a boolean
Keyword arguments:
arg -- The string to convert
Returns:
Boolean
"""
if isinstance(arg, bool):
return arg
elif isinstance(arg, str):
return arg.lower() in ["yes", "true", "on"]
else:
logger.debug("arg should be a boolean or a string, got %r", arg)
return True if arg else False
def unstable_apps(): def unstable_apps():
output = [] output = []
@ -2703,24 +2688,3 @@ def _assert_system_is_sane_for_app(manifest, when):
elif when == "post": elif when == "post":
raise YunohostError("this_action_broke_dpkg") raise YunohostError("this_action_broke_dpkg")
def human_to_binary(size: str) -> int:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
factor = {}
for i, s in enumerate(symbols):
factor[s] = 1 << (i + 1) * 10
suffix = size[-1]
size = size[:-1]
if suffix not in symbols:
raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}")
try:
size = float(size)
except Exception:
raise YunohostError(f"Failed to convert size {size} to float")
return size * factor[suffix]

View file

@ -39,9 +39,8 @@ from functools import reduce
from packaging import version from packaging import version
from moulinette import Moulinette, m18n from moulinette import Moulinette, m18n
from moulinette.utils import filesystem
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml from moulinette.utils.filesystem import read_file, mkdir, write_to_yaml, read_yaml, rm, chown, chmod
from moulinette.utils.process import check_output from moulinette.utils.process import check_output
import yunohost.domain import yunohost.domain
@ -67,8 +66,12 @@ from yunohost.tools import (
from yunohost.regenconf import regen_conf from yunohost.regenconf import regen_conf
from yunohost.log import OperationLogger, is_unit_operation from yunohost.log import OperationLogger, is_unit_operation
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.packages import ynh_packages_version from yunohost.utils.system import (
from yunohost.utils.filesystem import free_space_in_directory free_space_in_directory,
get_ynh_package_version,
binary_to_human,
space_used_by_directory,
)
from yunohost.settings import settings_get from yunohost.settings import settings_get
BACKUP_PATH = "/home/yunohost.backup" BACKUP_PATH = "/home/yunohost.backup"
@ -312,7 +315,7 @@ class BackupManager:
"size_details": self.size_details, "size_details": self.size_details,
"apps": self.apps_return, "apps": self.apps_return,
"system": self.system_return, "system": self.system_return,
"from_yunohost_version": ynh_packages_version()["yunohost"]["version"], "from_yunohost_version": get_ynh_package_version("yunohost")["version"],
} }
@property @property
@ -342,7 +345,7 @@ class BackupManager:
# FIXME replace isdir by exists ? manage better the case where the path # FIXME replace isdir by exists ? manage better the case where the path
# exists # exists
if not os.path.isdir(self.work_dir): if not os.path.isdir(self.work_dir):
filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin") mkdir(self.work_dir, 0o750, parents=True, uid="admin")
elif self.is_tmp_work_dir: elif self.is_tmp_work_dir:
logger.debug( logger.debug(
@ -357,8 +360,8 @@ class BackupManager:
# If umount succeeded, remove the directory (we checked that # If umount succeeded, remove the directory (we checked that
# we're in /home/yunohost.backup/tmp so that should be okay... # we're in /home/yunohost.backup/tmp so that should be okay...
# c.f. method clean() which also does this) # c.f. method clean() which also does this)
filesystem.rm(self.work_dir, recursive=True, force=True) rm(self.work_dir, recursive=True, force=True)
filesystem.mkdir(self.work_dir, 0o750, parents=True, uid="admin") mkdir(self.work_dir, 0o750, parents=True, uid="admin")
# #
# Backup target management # # Backup target management #
@ -535,7 +538,7 @@ class BackupManager:
successfull_system = self.targets.list("system", include=["Success", "Warning"]) successfull_system = self.targets.list("system", include=["Success", "Warning"])
if not successfull_apps and not successfull_system: if not successfull_apps and not successfull_system:
filesystem.rm(self.work_dir, True, True) rm(self.work_dir, True, True)
raise YunohostError("backup_nothings_done") raise YunohostError("backup_nothings_done")
# Add unlisted files from backup tmp dir # Add unlisted files from backup tmp dir
@ -647,7 +650,7 @@ class BackupManager:
restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore") restore_hooks_dir = os.path.join(self.work_dir, "hooks", "restore")
if not os.path.exists(restore_hooks_dir): if not os.path.exists(restore_hooks_dir):
filesystem.mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root") mkdir(restore_hooks_dir, mode=0o700, parents=True, uid="root")
restore_hooks = hook_list("restore")["hooks"] restore_hooks = hook_list("restore")["hooks"]
@ -714,7 +717,7 @@ class BackupManager:
tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app) tmp_workdir_for_app = _make_tmp_workdir_for_app(app=app)
try: try:
# Prepare backup directory for the app # Prepare backup directory for the app
filesystem.mkdir(tmp_app_bkp_dir, 0o700, True, uid="root") mkdir(tmp_app_bkp_dir, 0o700, True, uid="root")
# Copy the app settings to be able to call _common.sh # Copy the app settings to be able to call _common.sh
shutil.copytree(app_setting_path, settings_dir) shutil.copytree(app_setting_path, settings_dir)
@ -753,7 +756,7 @@ class BackupManager:
# Remove tmp files in all situations # Remove tmp files in all situations
finally: finally:
shutil.rmtree(tmp_workdir_for_app) shutil.rmtree(tmp_workdir_for_app)
filesystem.rm(env_dict["YNH_BACKUP_CSV"], force=True) rm(env_dict["YNH_BACKUP_CSV"], force=True)
# #
# Actual backup archive creation / method management # # Actual backup archive creation / method management #
@ -796,7 +799,7 @@ class BackupManager:
if row["dest"] == "info.json": if row["dest"] == "info.json":
continue continue
size = disk_usage(row["source"]) size = space_used_by_directory(row["source"], follow_symlinks=False)
# Add size to apps details # Add size to apps details
splitted_dest = row["dest"].split("/") splitted_dest = row["dest"].split("/")
@ -945,7 +948,7 @@ class RestoreManager:
ret = subprocess.call(["umount", self.work_dir]) ret = subprocess.call(["umount", self.work_dir])
if ret != 0: if ret != 0:
logger.warning(m18n.n("restore_cleaning_failed")) logger.warning(m18n.n("restore_cleaning_failed"))
filesystem.rm(self.work_dir, recursive=True, force=True) rm(self.work_dir, recursive=True, force=True)
# #
# Restore target manangement # # Restore target manangement #
@ -975,7 +978,7 @@ class RestoreManager:
available_restore_system_hooks = hook_list("restore")["hooks"] available_restore_system_hooks = hook_list("restore")["hooks"]
custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore") custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore")
filesystem.mkdir(custom_restore_hook_folder, 755, parents=True, force=True) mkdir(custom_restore_hook_folder, 755, parents=True, force=True)
for system_part in target_list: for system_part in target_list:
# By default, we'll use the restore hooks on the current install # By default, we'll use the restore hooks on the current install
@ -1080,7 +1083,7 @@ class RestoreManager:
else: else:
raise YunohostError("restore_removing_tmp_dir_failed") raise YunohostError("restore_removing_tmp_dir_failed")
filesystem.mkdir(self.work_dir, parents=True) mkdir(self.work_dir, parents=True)
self.method.mount() self.method.mount()
@ -1398,7 +1401,7 @@ class RestoreManager:
# Delete _common.sh file in backup # Delete _common.sh file in backup
common_file = os.path.join(app_backup_in_archive, "_common.sh") common_file = os.path.join(app_backup_in_archive, "_common.sh")
filesystem.rm(common_file, force=True) rm(common_file, force=True)
# Check if the app has a restore script # Check if the app has a restore script
app_restore_script_in_archive = os.path.join(app_scripts_in_archive, "restore") app_restore_script_in_archive = os.path.join(app_scripts_in_archive, "restore")
@ -1414,14 +1417,14 @@ class RestoreManager:
) )
app_scripts_new_path = os.path.join(app_settings_new_path, "scripts") app_scripts_new_path = os.path.join(app_settings_new_path, "scripts")
shutil.copytree(app_settings_in_archive, app_settings_new_path) shutil.copytree(app_settings_in_archive, app_settings_new_path)
filesystem.chmod(app_settings_new_path, 0o400, 0o400, True) chmod(app_settings_new_path, 0o400, 0o400, True)
filesystem.chown(app_scripts_new_path, "root", None, True) chown(app_scripts_new_path, "root", None, True)
# Copy the app scripts to a writable temporary folder # Copy the app scripts to a writable temporary folder
tmp_workdir_for_app = _make_tmp_workdir_for_app() tmp_workdir_for_app = _make_tmp_workdir_for_app()
copytree(app_scripts_in_archive, tmp_workdir_for_app) copytree(app_scripts_in_archive, tmp_workdir_for_app)
filesystem.chmod(tmp_workdir_for_app, 0o700, 0o700, True) chmod(tmp_workdir_for_app, 0o700, 0o700, True)
filesystem.chown(tmp_workdir_for_app, "root", None, True) chown(tmp_workdir_for_app, "root", None, True)
restore_script = os.path.join(tmp_workdir_for_app, "restore") restore_script = os.path.join(tmp_workdir_for_app, "restore")
# Restore permissions # Restore permissions
@ -1724,7 +1727,7 @@ class BackupMethod(object):
raise YunohostError("backup_cleaning_failed") raise YunohostError("backup_cleaning_failed")
if self.manager.is_tmp_work_dir: if self.manager.is_tmp_work_dir:
filesystem.rm(self.work_dir, True, True) rm(self.work_dir, True, True)
def _check_is_enough_free_space(self): def _check_is_enough_free_space(self):
""" """
@ -1772,11 +1775,11 @@ class BackupMethod(object):
# Be sure the parent dir of destination exists # Be sure the parent dir of destination exists
if not os.path.isdir(dest_dir): if not os.path.isdir(dest_dir):
filesystem.mkdir(dest_dir, parents=True) mkdir(dest_dir, parents=True)
# For directory, attempt to mount bind # For directory, attempt to mount bind
if os.path.isdir(src): if os.path.isdir(src):
filesystem.mkdir(dest, parents=True, force=True) mkdir(dest, parents=True, force=True)
try: try:
subprocess.check_call(["mount", "--rbind", src, dest]) subprocess.check_call(["mount", "--rbind", src, dest])
@ -1830,7 +1833,7 @@ class BackupMethod(object):
# to mounting error # to mounting error
# Compute size to copy # Compute size to copy
size = sum(disk_usage(path["source"]) for path in paths_needed_to_be_copied) size = sum(space_used_by_directory(path["source"], follow_symlinks=False) for path in paths_needed_to_be_copied)
size /= 1024 * 1024 # Convert bytes to megabytes size /= 1024 * 1024 # Convert bytes to megabytes
# Ask confirmation for copying # Ask confirmation for copying
@ -1882,7 +1885,7 @@ class CopyBackupMethod(BackupMethod):
dest_parent = os.path.dirname(dest) dest_parent = os.path.dirname(dest)
if not os.path.exists(dest_parent): if not os.path.exists(dest_parent):
filesystem.mkdir(dest_parent, 0o700, True, uid="admin") mkdir(dest_parent, 0o700, True, uid="admin")
if os.path.isdir(source): if os.path.isdir(source):
shutil.copytree(source, dest) shutil.copytree(source, dest)
@ -1900,7 +1903,7 @@ class CopyBackupMethod(BackupMethod):
if not os.path.isdir(self.repo): if not os.path.isdir(self.repo):
raise YunohostError("backup_no_uncompress_archive_dir") raise YunohostError("backup_no_uncompress_archive_dir")
filesystem.mkdir(self.work_dir, parent=True) mkdir(self.work_dir, parent=True)
ret = subprocess.call(["mount", "-r", "--rbind", self.repo, self.work_dir]) ret = subprocess.call(["mount", "-r", "--rbind", self.repo, self.work_dir])
if ret == 0: if ret == 0:
return return
@ -1944,7 +1947,7 @@ class TarBackupMethod(BackupMethod):
""" """
if not os.path.exists(self.repo): if not os.path.exists(self.repo):
filesystem.mkdir(self.repo, 0o750, parents=True, uid="admin") mkdir(self.repo, 0o750, parents=True, uid="admin")
# Check free space in output # Check free space in output
self._check_is_enough_free_space() self._check_is_enough_free_space()
@ -2667,32 +2670,3 @@ def _recursive_umount(directory):
continue continue
return everything_went_fine return everything_went_fine
def disk_usage(path):
# We don't do this in python with os.stat because we don't want
# to follow symlinks
du_output = check_output(["du", "-sb", path], shell=False)
return int(du_output.split()[0])
def binary_to_human(n, customary=False):
"""
Convert bytes or bits into human readable format with binary prefix
Keyword argument:
n -- Number to convert
customary -- Use customary symbol instead of IEC standard
"""
symbols = ("Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
if customary:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
prefix = {}
for i, s in enumerate(symbols):
prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
return "%.1f%s" % (value, s)
return "%s" % n

View file

@ -10,8 +10,8 @@ from moulinette.utils.filesystem import read_file
from yunohost.tools import Migration, tools_update, tools_upgrade from yunohost.tools import Migration, tools_update, tools_upgrade
from yunohost.app import unstable_apps from yunohost.app import unstable_apps
from yunohost.regenconf import manually_modified_files from yunohost.regenconf import manually_modified_files
from yunohost.utils.filesystem import free_space_in_directory from yunohost.utils.system import (
from yunohost.utils.packages import ( free_space_in_directory,
get_ynh_package_version, get_ynh_package_version,
_list_upgradable_apt_packages, _list_upgradable_apt_packages,
) )

View file

@ -5,7 +5,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from yunohost.tools import Migration from yunohost.tools import Migration
from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory from yunohost.utils.system import free_space_in_directory, space_used_by_directory
logger = getActionLogger("yunohost.migration") logger = getActionLogger("yunohost.migration")

View file

@ -38,7 +38,7 @@ from io import IOBase
from moulinette import m18n, Moulinette from moulinette import m18n, Moulinette
from moulinette.core import MoulinetteError from moulinette.core import MoulinetteError
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.packages import get_ynh_package_version from yunohost.utils.system import get_ynh_package_version
from moulinette.utils.log import getActionLogger from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, read_yaml from moulinette.utils.filesystem import read_file, read_yaml

View file

@ -48,10 +48,12 @@ from yunohost.domain import domain_add
from yunohost.firewall import firewall_upnp from yunohost.firewall import firewall_upnp
from yunohost.service import service_start, service_enable from yunohost.service import service_start, service_enable
from yunohost.regenconf import regen_conf from yunohost.regenconf import regen_conf
from yunohost.utils.packages import ( from yunohost.utils.system import (
_dump_sources_list, _dump_sources_list,
_list_upgradable_apt_packages, _list_upgradable_apt_packages,
ynh_packages_version, ynh_packages_version,
dpkg_is_broken,
dpkg_lock_available,
) )
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.log import is_unit_operation, OperationLogger from yunohost.log import is_unit_operation, OperationLogger
@ -171,20 +173,6 @@ def _set_hostname(hostname, pretty_hostname=None):
logger.debug(out) logger.debug(out)
def _detect_virt():
"""
Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...)
You can check the man of the command to have a list of possible outputs...
"""
p = subprocess.Popen(
"systemd-detect-virt".split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
out, _ = p.communicate()
return out.split()[0]
@is_unit_operation() @is_unit_operation()
def tools_postinstall( def tools_postinstall(
operation_logger, operation_logger,
@ -463,13 +451,12 @@ def tools_upgrade(
apps -- List of apps to upgrade (or [] to update all apps) apps -- List of apps to upgrade (or [] to update all apps)
system -- True to upgrade system system -- True to upgrade system
""" """
from yunohost.utils import packages
if packages.dpkg_is_broken(): if dpkg_is_broken():
raise YunohostValidationError("dpkg_is_broken") raise YunohostValidationError("dpkg_is_broken")
# Check for obvious conflict with other dpkg/apt commands already running in parallel # Check for obvious conflict with other dpkg/apt commands already running in parallel
if not packages.dpkg_lock_available(): if not dpkg_lock_available():
raise YunohostValidationError("dpkg_lock_not_available") raise YunohostValidationError("dpkg_lock_not_available")
# Legacy options management (--system, --apps) # Legacy options management (--system, --apps)

View file

@ -40,6 +40,7 @@ from moulinette.utils.process import check_output
from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.service import service_status from yunohost.service import service_status
from yunohost.log import is_unit_operation from yunohost.log import is_unit_operation
from yunohost.utils.system import binary_to_human
logger = getActionLogger("yunohost.user") logger = getActionLogger("yunohost.user")
@ -597,7 +598,7 @@ def user_info(username):
if has_value: if has_value:
storage_use = int(has_value.group(1)) storage_use = int(has_value.group(1))
storage_use = _convertSize(storage_use) storage_use = binary_to_human(storage_use)
if is_limited: if is_limited:
has_percent = re.search(r"%=(\d+)", cmd_result) has_percent = re.search(r"%=(\d+)", cmd_result)
@ -1330,15 +1331,6 @@ def user_ssh_remove_key(username, key):
# End SSH subcategory # End SSH subcategory
# #
def _convertSize(num, suffix=""):
for unit in ["K", "M", "G", "T", "P", "E", "Z"]:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, "Yi", suffix)
def _hash_user_password(password): def _hash_user_password(password):
""" """
This function computes and return a salted hash for the password in input. This function computes and return a salted hash for the password in input.

View file

@ -1,31 +0,0 @@
# -*- coding: utf-8 -*-
""" License
Copyright (C) 2018 YUNOHOST.ORG
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program; if not, see http://www.gnu.org/licenses
"""
import os
def free_space_in_directory(dirpath):
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_bavail
def space_used_by_directory(dirpath):
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_blocks

View file

@ -23,13 +23,85 @@ import os
import logging import logging
from moulinette.utils.process import check_output from moulinette.utils.process import check_output
from packaging import version from yunohost.utils.error import YunohostError
logger = logging.getLogger("yunohost.utils.packages") logger = logging.getLogger("yunohost.utils.packages")
YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"] YUNOHOST_PACKAGES = ["yunohost", "yunohost-admin", "moulinette", "ssowat"]
def system_arch():
return check_output("dpkg --print-architecture")
def system_virt():
"""
Returns the output of systemd-detect-virt (so e.g. 'none' or 'lxc' or ...)
You can check the man of the command to have a list of possible outputs...
"""
# Detect virt technology (if not bare metal) and arch
# Gotta have this "|| true" because it systemd-detect-virt return 'none'
# with an error code on bare metal ~.~
return check_output("systemd-detect-virt || true")
def free_space_in_directory(dirpath):
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_bavail
def space_used_by_directory(dirpath, follow_symlinks=True):
if not follow_symlinks:
du_output = check_output(["du", "-sb", dirpath], shell=False)
return int(du_output.split()[0])
stat = os.statvfs(dirpath)
return stat.f_frsize * stat.f_blocks
def human_to_binary(size: str) -> int:
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
factor = {}
for i, s in enumerate(symbols):
factor[s] = 1 << (i + 1) * 10
suffix = size[-1]
size = size[:-1]
if suffix not in symbols:
raise YunohostError(f"Invalid size suffix '{suffix}', expected one of {symbols}")
try:
size = float(size)
except Exception:
raise YunohostError(f"Failed to convert size {size} to float")
return size * factor[suffix]
def binary_to_human(n: int) -> str:
"""
Convert bytes or bits into human readable format with binary prefix
"""
symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y")
prefix = {}
for i, s in enumerate(symbols):
prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
return "%.1f%s" % (value, s)
return "%s" % n
def ram_available():
import psutil
return (psutil.virtual_memory().available, psutil.swap_memory().free)
def get_ynh_package_version(package): def get_ynh_package_version(package):
# Returns the installed version and release version ('stable' or 'testing' # Returns the installed version and release version ('stable' or 'testing'
@ -48,43 +120,6 @@ def get_ynh_package_version(package):
return {"version": out[1].strip("()"), "repo": out[2].strip(";")} return {"version": out[1].strip("()"), "repo": out[2].strip(";")}
def meets_version_specifier(pkg_name, specifier):
"""
Check if a package installed version meets specifier
specifier is something like ">> 1.2.3"
"""
# In practice, this function is only used to check the yunohost version
# installed.
# We'll trim any ~foobar in the current installed version because it's not
# handled correctly by version.parse, but we don't care so much in that
# context
assert pkg_name in YUNOHOST_PACKAGES
pkg_version = get_ynh_package_version(pkg_name)["version"]
pkg_version = re.split(r"\~|\+|\-", pkg_version)[0]
pkg_version = version.parse(pkg_version)
# Extract operator and version specifier
op, req_version = re.search(r"(<<|<=|=|>=|>>) *([\d\.]+)", specifier).groups()
req_version = version.parse(req_version)
# Python2 had a builtin that returns (-1, 0, 1) depending on comparison
# c.f. https://stackoverflow.com/a/22490617
def cmp(a, b):
return (a > b) - (a < b)
deb_operators = {
"<<": lambda v1, v2: cmp(v1, v2) in [-1],
"<=": lambda v1, v2: cmp(v1, v2) in [-1, 0],
"=": lambda v1, v2: cmp(v1, v2) in [0],
">=": lambda v1, v2: cmp(v1, v2) in [0, 1],
">>": lambda v1, v2: cmp(v1, v2) in [1],
}
return deb_operators[op](pkg_version, req_version)
def ynh_packages_version(*args, **kwargs): def ynh_packages_version(*args, **kwargs):
# from cli the received arguments are: # from cli the received arguments are:
# (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {} # (Namespace(_callbacks=deque([]), _tid='_global', _to_return={}), []) {}