From 8494485fce404f822c0696324b63f54d3c652fd9 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Fri, 5 Mar 2021 05:00:42 +0100 Subject: [PATCH] Implement ynh_install_python --- README.md | 3 - scripts/_common.sh | 2 + scripts/backup | 3 - scripts/change_url | 1 + scripts/install | 29 ++-- scripts/remove | 5 +- scripts/restore | 30 ++++- scripts/upgrade | 44 ++++++- scripts/ynh_install_python | 264 +++++++++++++++++++++++++++++++++++++ 9 files changed, 354 insertions(+), 27 deletions(-) create mode 100644 scripts/ynh_install_python diff --git a/README.md b/README.md index 5cab32b..140c5a2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/install) to learn how to install it.* ## Overview - Pepettes is a donation form based on Stripe. **Shipped version:** 1.0 @@ -54,7 +53,6 @@ Can the app be used by multiple users? yes ## Links * Report a bug: https://github.com/YunoHost-Apps/pepettes_ynh/issues - * App website: - * Upstream app repository: https://github.com/YunoHost/pepettes/ * YunoHost website: https://yunohost.org/ @@ -62,7 +60,6 @@ Can the app be used by multiple users? yes ## Developer info -**Only if you want to use a testing branch for coding, instead of merging directly into master.** Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/pepettes_ynh/tree/testing). To try the testing branch, please proceed like that. diff --git a/scripts/_common.sh b/scripts/_common.sh index bc48e71..ea69553 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -7,6 +7,8 @@ # dependencies used by the app pkg_dependencies="python3-venv python3-dev python3-pip" +python_version=3.9.2 + #================================================= # PERSONAL HELPERS #================================================= diff --git a/scripts/backup b/scripts/backup index 1400566..3beb939 100755 --- a/scripts/backup +++ b/scripts/backup @@ -15,7 +15,6 @@ source /usr/share/yunohost/helpers #================================================= ynh_clean_setup () { - ### Remove this function if there's nothing to clean before calling the remove script. true } # Exit if an error occurs during the execution of the script @@ -51,8 +50,6 @@ ynh_backup --src_path="/etc/nginx/conf.d/$domain.d/$app.conf" #================================================= # SPECIFIC BACKUP -#================================================= - #================================================= # BACKUP SYSTEMD #================================================= diff --git a/scripts/change_url b/scripts/change_url index 17802c4..04784e9 100644 --- a/scripts/change_url +++ b/scripts/change_url @@ -118,6 +118,7 @@ fi #================================================= ynh_script_progression --message="Starting a systemd service..." --weight=1 +# Start a systemd service ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$app/$app.log" #================================================= diff --git a/scripts/install b/scripts/install index 1e50915..facc68b 100755 --- a/scripts/install +++ b/scripts/install @@ -7,6 +7,7 @@ #================================================= source _common.sh +source ynh_install_python source /usr/share/yunohost/helpers #================================================= @@ -14,7 +15,6 @@ source /usr/share/yunohost/helpers #================================================= ynh_clean_setup () { - ### Remove this function if there's nothing to clean before calling the remove script. true } # Exit if an error occurs during the execution of the script @@ -84,6 +84,7 @@ ynh_app_setting_set --app=$app --key=port --value=$port ynh_script_progression --message="Installing dependencies..." --weight=1 ynh_install_app_dependencies $pkg_dependencies +ynh_install_python --python_version=$python_version #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE @@ -113,20 +114,23 @@ ynh_system_user_create --username=$app #================================================= # SPECIFIC SETUP #================================================= +# INSTALL PYTHON DEPENDENCIES +#================================================= +ynh_script_progression --message="Installing Python dependencies..." + pushd $final_path -python3 -m venv venv -venv/bin/pip install -r requirements.txt -venv/bin/pip install gunicorn -mkdir -p /var/log/$app -chown -R $app:www-data /var/log/$app -cat <> wsgi.py + ynh_use_python + $ynh_pip install -r requirements.txt + $ynh_pip install gunicorn + mkdir -p /var/log/$app + chown -R $app:www-data /var/log/$app + cat <> wsgi.py from server import app if __name__ == "__main__": app.run() EOF popd -#================================================= #================================================= # SETUP SYSTEMD @@ -139,17 +143,11 @@ ynh_add_systemd_config #================================================= # MODIFY A CONFIG FILE #================================================= +ynh_script_progression --message="Modifying a config file..." ynh_add_config --template="../conf/gunicorn.py" --destination="$final_path/gunicorn.py" ynh_add_config --template="../conf/settings.py" --destination="$final_path/settings.py" -#================================================= -# STORE THE CONFIG FILE CHECKSUM -#================================================= - -# Calculate and store the config file checksum into the app settings -ynh_store_file_checksum --file="$final_path/gunicorn.py" -ynh_store_file_checksum --file="$final_path/settings.py" for price in $(echo $prices | sed "s/,/ /"); do frequency=$(echo $price | cut -d/ -f1) currency=$(echo $price | cut -d/ -f2) @@ -162,6 +160,7 @@ done #================================================= # SECURE FILES AND DIRECTORIES #================================================= +ynh_script_progression --message="Securing files and directories..." # Set permissions to app files chown -R root: $final_path diff --git a/scripts/remove b/scripts/remove index a2cf263..feedb2a 100755 --- a/scripts/remove +++ b/scripts/remove @@ -7,6 +7,7 @@ #================================================= source _common.sh +source ynh_install_python source /usr/share/yunohost/helpers #================================================= @@ -47,6 +48,7 @@ ynh_remove_systemd_config ynh_script_progression --message="Removing dependencies..." --weight=1 # Remove metapackage and its dependencies +ynh_remove_python ynh_remove_app_dependencies #================================================= @@ -70,9 +72,10 @@ ynh_remove_nginx_config #================================================= # REMOVE VARIOUS FILES #================================================= +ynh_script_progression --message="Removing various files..." # Remove the log files -#ynh_secure_remove --file="/var/log/$app/" +ynh_secure_remove --file="/var/log/$app" #================================================= # GENERIC FINALIZATION diff --git a/scripts/restore b/scripts/restore index a8fc3ba..2fe5759 100755 --- a/scripts/restore +++ b/scripts/restore @@ -8,6 +8,7 @@ # Keep this path for calling _common.sh inside the execution's context of backup and restore scripts source ../settings/scripts/_common.sh +source ../settings/scripts/ynh_install_python source /usr/share/yunohost/helpers #================================================= @@ -15,7 +16,6 @@ source /usr/share/yunohost/helpers #================================================= ynh_clean_setup () { - #### Remove this function if there's nothing to clean before calling the remove script. true } # Exit if an error occurs during the execution of the script @@ -24,7 +24,7 @@ ynh_abort_if_errors #================================================= # LOAD SETTINGS #================================================= -ynh_script_progression --message="Loading installation settings..." --weight=1 +ynh_script_progression --message="Loading settings..." --weight=1 app=$YNH_APP_INSTANCE_NAME @@ -47,6 +47,7 @@ test ! -d $final_path \ #================================================= # RESTORE THE NGINX CONFIGURATION #================================================= +ynh_script_progression --message="Restoring the NGINX web server configuration..." ynh_restore_file --origin_path="/etc/nginx/conf.d/$domain.d/$app.conf" @@ -68,6 +69,7 @@ ynh_system_user_create --username=$app #================================================= # RESTORE USER RIGHTS #================================================= +ynh_script_progression --message="Restoring user rights..." # Restore permissions on app files chown -R root: $final_path @@ -81,6 +83,26 @@ ynh_script_progression --message="Reinstalling dependencies..." --weight=1 # Define and install dependencies ynh_install_app_dependencies $pkg_dependencies +ynh_install_python --python_version=$python_version + +#================================================= +# INSTALL PYTHON DEPENDENCIES +#================================================= +ynh_script_progression --message="Installing Python dependencies..." + +pushd $final_path + ynh_use_python + $ynh_pip install -r requirements.txt + $ynh_pip install gunicorn + mkdir -p /var/log/$app + chown -R $app:www-data /var/log/$app + cat <> wsgi.py +from server import app + +if __name__ == "__main__": + app.run() +EOF +popd #================================================= # RESTORE SYSTEMD @@ -107,9 +129,9 @@ ynh_systemd_action --service_name=$app --action="start" --log_path="/var/log/$ap #================================================= # GENERIC FINALIZATION #================================================= -# RELOAD NGINX AND PHP-FPM +# RELOAD NGINX #================================================= -ynh_script_progression --message="Reloading NGINX web server and PHP-FPM..." --weight=1 +ynh_script_progression --message="Reloading NGINX web server..." --weight=1 ynh_systemd_action --service_name=nginx --action=reload diff --git a/scripts/upgrade b/scripts/upgrade index 46dfb32..9025f28 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -7,6 +7,7 @@ #================================================= source _common.sh +source ynh_install_python source /usr/share/yunohost/helpers #================================================= @@ -20,9 +21,20 @@ domain=$(ynh_app_setting_get --app=$app --key=domain) path_url=$(ynh_app_setting_get --app=$app --key=path) final_path=$(ynh_app_setting_get --app=$app --key=final_path) +project_name=$(ynh_app_setting_get --app=$app --key=project_name) +contact_url=$(ynh_app_setting_get --app=$app --key=contact_url) +logo=$(ynh_app_setting_get --app=$app --key=logo) +favicon=$(ynh_app_setting_get --app=$app --key=favicon) +publishable_key=$(ynh_app_setting_get --app=$app --key=publishable_key) +secret_key=$(ynh_app_setting_get --app=$app --key=secret_key) +prices=$(ynh_app_setting_get --app=$app --key=prices) +secret=$(ynh_app_setting_get --app=$app --key=secret) +csrf_key=$(ynh_app_setting_get --app=$app --key=csrf_key) + #================================================= # CHECK VERSION #================================================= +ynh_script_progression --message="Checking version..." upgrade_type=$(ynh_check_app_version_changed) @@ -80,6 +92,7 @@ ynh_add_nginx_config ynh_script_progression --message="Upgrading dependencies..." --weight=1 ynh_install_app_dependencies $pkg_dependencies +ynh_install_python --python_version=$python_version #================================================= # CREATE DEDICATED USER @@ -92,8 +105,23 @@ ynh_system_user_create --username=$app #================================================= # SPECIFIC UPGRADE #================================================= -# ... +# UPGRADE PYTHON DEPENDENCIES #================================================= +ynh_script_progression --message="Installing Python dependencies..." + +pushd $final_path + ynh_use_python + $ynh_pip install -r requirements.txt + $ynh_pip install gunicorn + mkdir -p /var/log/$app + chown -R $app:www-data /var/log/$app + cat <> wsgi.py +from server import app + +if __name__ == "__main__": + app.run() +EOF +popd #================================================= # SETUP SYSTEMD @@ -106,16 +134,30 @@ ynh_add_systemd_config #================================================= # MODIFY A CONFIG FILE #================================================= +ynh_script_progression --message="Modifying a config file..." +ynh_add_config --template="../conf/gunicorn.py" --destination="$final_path/gunicorn.py" +ynh_add_config --template="../conf/settings.py" --destination="$final_path/settings.py" + +for price in $(echo $prices | sed "s/,/ /"); do + frequency=$(echo $price | cut -d/ -f1) + currency=$(echo $price | cut -d/ -f2) + price_id=$(echo $price | cut -d/ -f3) + echo "DONATION['$frequency']['$currency'] = '$price_id'" >> "$final_path/settings.py" +done #================================================= # GENERIC FINALIZATION #================================================= # SECURE FILES AND DIRECTORIES #================================================= +ynh_script_progression --message="Securing files and directories..." # Set permissions on app files chown -R root: $final_path +chown pepettes:root $final_path +chown pepettes:root $final_path/settings.py +chmod o=--- $final_path/settings.py #================================================= # INTEGRATE SERVICE IN YUNOHOST diff --git a/scripts/ynh_install_python b/scripts/ynh_install_python new file mode 100644 index 0000000..910aaa2 --- /dev/null +++ b/scripts/ynh_install_python @@ -0,0 +1,264 @@ +#!/bin/bash + +pyenv_version=1.2.23 +pyenv_virtualenv_version=1.1.5 +pyenv_install_dir="/opt/pyenv" +python_version_path="$pyenv_install_dir/versions" +# PYENV_ROOT is the directory of pyenv, it needs to be loaded as a environment variable. +export PYENV_ROOT="$pyenv_install_dir" + +# Required dependencies +pyenv_dependencies="build-essential libssl1.0-dev|libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl git" + +# Install Python Version Management +# +# [internal] +# +# usage: ynh_install_pyenv +# +# Requires YunoHost version 2.7.12 or higher. +ynh_install_pyenv () { + ynh_print_info --message="Installation of pyenv - Python Version Management - pyenv-$pyenv_version/pyenv-virtualenv-$pyenv_virtualenv_version" + + # Build an app.src for pyenv + mkdir -p "../conf" + echo "SOURCE_URL=https://github.com/pyenv/pyenv/archive/v${pyenv_version}.tar.gz +SOURCE_SUM=805058aa5ce257157fb4769543e6a43bac45a88c723ff3c4fcf5b4f759056bf5" > "../conf/pyenv.src" + # Download and extract pyenv + ynh_setup_source --dest_dir="$pyenv_install_dir" --source_id=pyenv + + # Build an app.src for pyenv-virtualenv + mkdir -p "../conf" + echo "SOURCE_URL=https://github.com/pyenv/pyenv-virtualenv/archive/v${pyenv_virtualenv_version}.tar.gz +SOURCE_SUM=27ae3de027a6f6dccdca4085225512e559c6b94b31625bd2b357a18890a1e618" > "../conf/pyenv-virtualenv.src" + # Download and extract pyenv-virtualenv + ynh_setup_source --dest_dir="$pyenv_install_dir/plugins/pyenv-virtualenv" --source_id=pyenv-virtualenv + + (cd $pyenv_install_dir + ./src/configure && make -C src) + +# Create shims directory if needed +if [ ! -d $pyenv_install_dir/shims ] ; then + mkdir $pyenv_install_dir/shims +fi +} + +# Load the version of python for an app, and set variables. +# +# ynh_use_python has to be used in any app scripts before using python for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use pip or python, use the alias `ynh_pip` and `ynh_python` +# Those alias will use the correct version installed for the app +# For example: use `ynh_pip install` instead of `pip install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_pip` and `$ynh_python` +# And propagate $PATH to sudo with $ynh_python_load_path +# Exemple: `ynh_exec_as $app $ynh_python_load_path $ynh_pip install` +# +# $PATH contains the path of the requested version of python. +# However, $PATH is duplicated into $python_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_python_load_path` to quickly load your python version +# in $PATH for an usage into a separate script. +# Exemple: $ynh_python_load_path $final_path/script_that_use_pip.sh` +# +# +# Finally, to start a python service with the correct version, 2 solutions +# Either the app is dependent of python or pip, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_PYTHON_LOAD_ENV_PATH__"` +# `ExecStart=__FINALPATH__/my_app` +# You will replace __YNH_PYTHON_LOAD_ENV_PATH__ with $ynh_python_load_path +# +# Or python start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_PYTHON__ my_app run` +# You will replace __YNH_PYTHON__ with $ynh_python +# +# +# one other variable is also available +# - $python_path: The absolute path to python binaries for the chosen version. +# +# usage: ynh_use_python +# +# Requires YunoHost version 2.7.12 or higher. +ynh_use_python () { + # Get the absolute path of this version of python + python_path="$python_version_path/$YNH_APP_INSTANCE_NAME/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of python and a variable as fallback + ynh_python="$python_path/python" + alias ynh_python="$ynh_python" + # And pip + ynh_pip="$python_path/pip" + alias ynh_pip="$ynh_pip" + + # Load the path of this version of python in $PATH + if [[ :$PATH: != *":$python_path"* ]]; then + PATH="$python_path:$PATH" + fi + python_path="$PATH" + # Create an alias to easily load the PATH + ynh_python_load_path="PATH=$python_path" + + # Sets the local application-specific Python version + (cd $final_path + pyenv local $python_version) +} + +# Install a specific version of Python +# +# ynh_install_python will install the version of Python provided as argument by using pyenv. +# +# pyenv (Python Version Management) stores the target Python version in a .python_version file created in the target folder (using pyenv local ) +# It then uses that information for every Python user that uses pyenv provided Python command +# +# This helper creates a /etc/profile.d/pyenv.sh that configures PATH environment for pyenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute python-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to pyenv shims (e.g. $PYENV_ROOT/shims/bundle) +# +# usage: ynh_install_python --python_version=python_version +# | arg: -v, --python_version= - Version of Python to install. +# +# Requires YunoHost version 2.7.12 or higher. +ynh_install_python () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=python_version= ) + local python_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Store python_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=python_version --value=$python_version + + # Create $pyenv_install_dir if doesn't exist already + mkdir -p "$pyenv_install_dir/plugins/pyenv-virtualenv" + + # Install required dependencies + ynh_add_app_dependencies --package="$pyenv_dependencies" + + # Load pyenv path in PATH + local CLEAR_PATH="$pyenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Python prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Python binary, to avoid to block pyenv + #test -x /usr/bin/python && mv /usr/bin/python /usr/bin/python_pyenv + + # If pyenv is not previously setup, install it + if ! type pyenv > /dev/null 2>&1 + then + ynh_install_pyenv + elif dpkg --compare-versions "$($pyenv_install_dir/bin/pyenv --version | cut -d" " -f2)" lt $pyenv_version + then + ynh_install_pyenv + elif dpkg --compare-versions "$($pyenv_install_dir/bin/pyenv virtualenv --version | cut -d" " -f2)" lt $pyenv_virtualenv_version + then + ynh_install_pyenv + fi + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Python binary + # test -x /usr/bin/python_pyenv && mv /usr/bin/python_pyenv /usr/bin/python + + # Install the requested version of Python + ynh_print_info --message="Installation of Python-"$python_version + pyenv install --skip-existing $python_version &>/dev/null + + # Remove app virtualenv + if $(pyenv virtualenvs | grep $YNH_APP_INSTANCE_NAME) + then + pyenv virtualenv-delete --force $YNH_APP_INSTANCE_NAME + fi + + # Create app virtualenv + pyenv virtualenv --force $python_version $YNH_APP_INSTANCE_NAME + + # Cleanup Python versions + ynh_cleanup_python + + # Set environment for Python users + echo "#pyenv +export PYENV_ROOT=$pyenv_install_dir +export PATH=\"$pyenv_install_dir/bin:$PATH\" +eval \"\$(pyenv init -)\" +#pyenv" > /etc/profile.d/pyenv.sh + + # Load the environment + eval "$(pyenv init -)" +} + +# Remove the version of Python used by the app. +# +# This helper will also cleanup Python versions +# +# usage: ynh_remove_python +ynh_remove_python () { + local python_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=python_version) + + # Load pyenv path in PATH + local CLEAR_PATH="$pyenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of python prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + pyenv virtualenv-delete --force $YNH_APP_INSTANCE_NAME + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=python_version + + # Cleanup Python versions + ynh_cleanup_python +} + +# Remove no more needed versions of Python used by the app. +# +# This helper will check what Python version are no more required, +# and uninstall them +# If no app uses Python, pyenv will be also removed. +# +# usage: ynh_cleanup_python +ynh_cleanup_python () { + + # List required Python version + local installed_apps=$(yunohost app list | grep -oP 'id: \K.*$') + local required_python_versions="" + for installed_app in $installed_apps + do + local installed_app_python_version=$(yunohost app setting $installed_app python_version) + if [[ $installed_app_python_version ]] + then + required_python_versions="${installed_app_python_version}\n${required_python_versions}" + fi + done + + # Remove no more needed Python version + local installed_python_versions=$(pyenv versions --bare --skip-aliases | grep -Ev '/') + for installed_python_version in $installed_python_versions + do + if ! `echo ${required_python_versions} | grep "${installed_python_version}" 1>/dev/null 2>&1` + then + ynh_print_info --message="Removing of Python-"$installed_python_version + $pyenv_install_dir/bin/pyenv uninstall --force $installed_python_version + fi + done + + # If none Python version is required + if [[ ! $required_python_versions ]] + then + # Remove pyenv environment configuration + ynh_print_info --message="Removing of pyenv-"$pyenv_version + ynh_secure_remove --file="$pyenv_install_dir" + rm /etc/profile.d/pyenv.sh + fi +} \ No newline at end of file