From 89034effd27b3312b2930823be38f64c9a2dfe7b Mon Sep 17 00:00:00 2001 From: Jules-Bertholet Date: Wed, 31 Mar 2021 02:04:35 -0400 Subject: [PATCH] Don't run build steps as root Who knows that's in those npm/pip dependencies? --- check_process | 8 +++--- conf/librephotos.env | 1 + hooks/post_user_create | 7 ++--- hooks/post_user_delete | 7 ++--- scripts/_common.sh | 2 +- scripts/install | 64 ++++++++++++++++++++++++------------------ scripts/restore | 4 +-- scripts/upgrade | 58 +++++++++++++++++++++----------------- 8 files changed, 83 insertions(+), 68 deletions(-) diff --git a/check_process b/check_process index 62d0b95..964d604 100644 --- a/check_process +++ b/check_process @@ -17,7 +17,7 @@ setup_private=1 setup_public=1 upgrade=1 - upgrade=0 from_commit=CommitHash + upgrade=1 from_commit=3b5b527e26cb6b530bd51f14c791d1dc15757af4 backup_restore=1 multi_instance=1 port_already_use=0 @@ -26,6 +26,6 @@ Email= Notification=none ;;; Upgrade options - ; commit=CommitHash - name=Name and date of the commit. - manifest_arg=domain=DOMAIN&admin=USER&is_public=1&allow_multimedia_write=1& + ; commit=3b5b527e26cb6b530bd51f14c791d1dc15757af4 + name=Upgrade from March 29 + manifest_arg=domain=domain.tld&admin=john&is_public=1&allow_multimedia_write=1& diff --git a/conf/librephotos.env b/conf/librephotos.env index a2fe721..8156452 100644 --- a/conf/librephotos.env +++ b/conf/librephotos.env @@ -23,5 +23,6 @@ MAPBOX_API_KEY= HEAVYWEIGHT_PROCESS=1 httpPort=__PORT__ +BACKEND_PORT=__BACKEND_PORT__ IMAGE_SIMILARITY_SERVER_PORT=__SIMILARITY_PORT__ MPLCONFIGDIR=__DATA_PATH__/matplotlib diff --git a/hooks/post_user_create b/hooks/post_user_create index 39e57b5..39f1d0f 100644 --- a/hooks/post_user_create +++ b/hooks/post_user_create @@ -2,15 +2,14 @@ set -a source /usr/share/yunohost/helpers -app="${0//50-}" +app="${0//.\/50-}" username=$1 mail=$2 final_path=$(ynh_app_setting_get --app=$app --key=final_path) pushd $final_path/backend - source $final_path/backend/venv/bin/activate - $final_path/bin/librephotos-cli shell <<< " + sudo -u $app $final_path/bin/librephotos-cli shell <<< " from django.contrib.auth import get_user_model User = get_user_model() try: @@ -19,5 +18,5 @@ try: user.save() except User.DoesNotExist: User.objects.create_user('$username', email='$mail', scan_directory='/home/yunohost.multimedia/$user/Picture') -" +" 2>&1 popd diff --git a/hooks/post_user_delete b/hooks/post_user_delete index c45266e..a4bd9dc 100644 --- a/hooks/post_user_delete +++ b/hooks/post_user_delete @@ -2,15 +2,14 @@ set -a source /usr/share/yunohost/helpers -app="${0//50-}" +app="${0//.\/50-}" username=$1 purge=$2 final_path=$(ynh_app_setting_get --app=$app --key=final_path) pushd $final_path/backend - source $final_path/backend/venv/bin/activate - $final_path/bin/librephotos-cli shell <<< " + sudo -u $app $final_path/bin/librephotos-cli shell <<< " from django.contrib.auth import get_user_model User = get_user_model() try: @@ -21,5 +20,5 @@ try: user.delete() except User.DoesNotExist: pass -" +" 2>&1 popd diff --git a/scripts/_common.sh b/scripts/_common.sh index c414b6e..431d04e 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,7 +5,7 @@ #================================================= # dependencies used by the app -pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch" +pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch yarnpkg" #================================================= # PERSONAL HELPERS diff --git a/scripts/install b/scripts/install index dd4a94a..1e249c3 100755 --- a/scripts/install +++ b/scripts/install @@ -140,16 +140,17 @@ usermod -d $data_path $app 2>&1 #================================================= ynh_script_progression --message="Initializing backend Python virtualenv..." --weight=60 -pushd $final_path/backend || ynh_die - python3 -m venv $final_path/backend/venv - source $final_path/backend/venv/bin/activate - python3 -m pip install -U wheel pip setuptools - python3 -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - python3 -m pip install --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib - python3 -m pip install --requirement $final_path/backend/requirements.txt - python3 -m pip install --requirement $final_path/backend/requirements-ynh.txt - python3 -m spacy download en_core_web_sm - deactivate 'dummy_arg' +backend_path=$final_path/backend +pushd $backend_path || ynh_die + chown -R $app:$app $backend_path + sudo -u $app python3 -m venv $backend_path/venv + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 + sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 + chown -R root:root $backend_path popd || ynh_die #================================================= @@ -157,13 +158,14 @@ popd || ynh_die #================================================= ynh_script_progression --message="Setting up frontend..." --weight=5 +frontend_path=$final_path/frontend pushd $final_path/frontend || ynh_die - npm install npm@latest --quiet 2>&1 - npx npm cache verify --quiet 2>&1 - npx npm install --quiet 2>&1 - npx npm install d3-zoom d3-force --quiet 2>&1 - SKIP_PREFLIGHT_CHECK=true npx npm run build --quiet 2>&1 - npx npm install serve --quiet 2>&1 + chown -R $app:$app $frontend_path + sudo -u $app touch $frontend_path/.yarnrc + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 + chown -R root:root $frontend_path popd || ynh_die #================================================= @@ -186,15 +188,20 @@ secret_key=$(ynh_string_random -l 64) ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" for file in $final_path/bin/*; do + ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file + echo "$(uniq $file)" > $file ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r $port -f $file - ynh_replace_string -m 8001 -r $backend_port -f $file - ynh_replace_string -m 8002 -r $similarity_port -f $file + ynh_replace_string -m 3000 -r '$httpPort' -f $file + ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file + ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_store_file_checksum --file="$file" + ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file + ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file done #================================================= @@ -203,13 +210,13 @@ done ynh_script_progression --message="Finalizing database..." --weight=1 pushd $final_path || ynh_die - python3 -m venv $final_path/backend/venv - source $final_path/backend/venv/bin/activate - $final_path/bin/librephotos-upgrade - $final_path/bin/librephotos-createadmin "$admin" "$(ynh_user_get_info $admin 'mail')" $(ynh_string_random -l 64) + chown -R $app:$app $final_path + chown -R $app:$app /var/log/$app + sudo -u $app $final_path/bin/librephotos-upgrade 2>&1 + sudo -u $app $final_path/bin/librephotos-createadmin "$admin" "$(ynh_user_get_info $admin 'mail')" $(ynh_string_random -l 64) 2>&1 for username in $(ynh_user_list); do user_email=$(ynh_user_get_info --username="$username" --key=mail) - $final_path/bin/librephotos-cli shell <<< " + sudo -u $app $final_path/bin/librephotos-cli shell <<< " from django.contrib.auth import get_user_model User = get_user_model() try: @@ -218,8 +225,9 @@ try: user.save() except User.DoesNotExist: User.objects.create_user('$username', email='$user_email', scan_directory='/home/yunohost.multimedia/$username/Picture') -" +" 2>&1 done + chown -R root:root $final_path popd || ynh_die #================================================= @@ -266,9 +274,9 @@ ynh_use_logrotate --logfile="/var/log/$app/ownphotos.log" --specific_user="$app/ #================================================= ynh_script_progression --message="Integrating services in YunoHost..." --weight=1 -yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/$app-backend.log" +yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/gunicorn_django.log" yunohost service add $app-frontend --description="Frontend for librephotos" --log="/var/log/$app/$app-frontend.log" -yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/$app-image-similarity.log" +yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/image_similarity.log" yunohost service add $app-worker --description="Worker for librephotos" --log="/var/log/$app/$app-worker.log" #================================================= diff --git a/scripts/restore b/scripts/restore index a9d8fd5..5372f67 100755 --- a/scripts/restore +++ b/scripts/restore @@ -153,9 +153,9 @@ systemctl enable $app-worker.service --quiet #================================================= ynh_script_progression --message="Integrating services in YunoHost..." --weight=1 -yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/$app-backend.log" +yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/gunicorn_django.log" yunohost service add $app-frontend --description="Frontend for librephotos" --log="/var/log/$app/$app-frontend.log" -yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/$app-image-similarity.log" +yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/image_similarity.log" yunohost service add $app-worker --description="Worker for librephotos" --log="/var/log/$app/$app-worker.log" #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 5ea9972..8eaff83 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -125,16 +125,17 @@ usermod -d $data_path $app 2>&1 #================================================= ynh_script_progression --message="Upgrading backend Python virtualenv..." --weight=50 -pushd $final_path/backend || ynh_die - python3 -m venv $final_path/backend/venv - source $final_path/backend/venv/bin/activate - python3 -m pip install -U wheel pip setuptools - python3 -m pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - python3 -m pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib - python3 -m pip install -U --requirement $final_path/backend/requirements.txt - python3 -m pip install -U --requirement $final_path/backend/requirements-ynh.txt - python3 -m spacy download en_core_web_sm - deactivate 'dummy_arg' +backend_path=$final_path/backend +pushd $backend_path || ynh_die + chown -R $app:$app $backend_path + sudo -u $app python3 -m venv $backend_path/venv + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 + sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 + chown -R root:root $backend_path popd || ynh_die #================================================= @@ -142,13 +143,14 @@ popd || ynh_die #================================================= ynh_script_progression --message="Setting up frontend..." --weight=5 +frontend_path=$final_path/frontend pushd $final_path/frontend || ynh_die - npm install npm@latest --quiet 2>&1 - npx npm cache verify --quiet 2>&1 - npx npm install --quiet 2>&1 - npx npm install d3-zoom d3-force --quiet 2>&1 - SKIP_PREFLIGHT_CHECK=true npx npm run build --quiet 2>&1 - npx npm install serve --quiet 2>&1 + chown -R $app:$app $frontend_path + sudo -u $app touch $frontend_path/.yarnrc + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 + chown -R root:root $frontend_path popd || ynh_die #================================================= @@ -171,15 +173,20 @@ secret_key=$(ynh_string_random -l 64) ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" for file in $final_path/bin/*; do + ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file + echo "$(uniq $file)" > $file ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r $port -f $file - ynh_replace_string -m 8001 -r $backend_port -f $file - ynh_replace_string -m 8002 -r $similarity_port -f $file + ynh_replace_string -m 3000 -r '$httpPort' -f $file + ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file + ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_store_file_checksum --file="$file" + ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file + ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file done #================================================= @@ -188,9 +195,10 @@ done ynh_script_progression --message="Finalizing database..." --weight=1 pushd $final_path || ynh_die - python3 -m venv $final_path/backend/venv - source $final_path/backend/venv/bin/activate - $final_path/bin/librephotos-upgrade + chown -R $app:$app $final_path + chown -R $app:$app /var/log/$app + sudo -u $app $final_path/bin/librephotos-upgrade 2>&1 + chown -R root:root $final_path popd || ynh_die #================================================= @@ -237,9 +245,9 @@ ynh_use_logrotate --logfile="/var/log/$app/ownphotos.log" --specific_user="$app/ #================================================= ynh_script_progression --message="Integrating services in YunoHost..." --weight=1 -yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/$app-backend.log" +yunohost service add $app-backend --description="Backend for librephotos" --log="/var/log/$app/gunicorn_django.log" yunohost service add $app-frontend --description="Frontend for librephotos" --log="/var/log/$app/$app-frontend.log" -yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/$app-image-similarity.log" +yunohost service add $app-image-similarity --description="Image similarity server for librephotos" --log="/var/log/$app/image_similarity.log" yunohost service add $app-worker --description="Worker for librephotos" --log="/var/log/$app/$app-worker.log" #=================================================