From dfa6cc60e147e388734de59ea5222e4e6116e993 Mon Sep 17 00:00:00 2001 From: Jules Bertholet Date: Tue, 30 Mar 2021 11:02:54 -0400 Subject: [PATCH] Don't run NPM as root Who knows what's in those dependencies --- conf/librephotos.env | 1 + hooks/post_user_create | 2 +- hooks/post_user_delete | 2 +- scripts/_common.sh | 2 +- scripts/install | 58 ++++++++++++++++++++++-------------------- scripts/upgrade | 54 +++++++++++++++++++++------------------ 6 files changed, 64 insertions(+), 55 deletions(-) diff --git a/conf/librephotos.env b/conf/librephotos.env index a2fe721..8156452 100644 --- a/conf/librephotos.env +++ b/conf/librephotos.env @@ -23,5 +23,6 @@ MAPBOX_API_KEY= HEAVYWEIGHT_PROCESS=1 httpPort=__PORT__ +BACKEND_PORT=__BACKEND_PORT__ IMAGE_SIMILARITY_SERVER_PORT=__SIMILARITY_PORT__ MPLCONFIGDIR=__DATA_PATH__/matplotlib diff --git a/hooks/post_user_create b/hooks/post_user_create index b7d523b..9c5e516 100644 --- a/hooks/post_user_create +++ b/hooks/post_user_create @@ -18,5 +18,5 @@ try: user.save() except User.DoesNotExist: User.objects.create_user('$username', email='$mail', scan_directory='/home/yunohost.multimedia/$user/Picture') -" +" 2>&1 popd diff --git a/hooks/post_user_delete b/hooks/post_user_delete index e85f678..74cea86 100644 --- a/hooks/post_user_delete +++ b/hooks/post_user_delete @@ -20,5 +20,5 @@ try: user.delete() except User.DoesNotExist: pass -" +" 2>&1 popd diff --git a/scripts/_common.sh b/scripts/_common.sh index c414b6e..431d04e 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,7 +5,7 @@ #================================================= # dependencies used by the app -pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch" +pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch yarnpkg" #================================================= # PERSONAL HELPERS diff --git a/scripts/install b/scripts/install index 8632934..1e249c3 100755 --- a/scripts/install +++ b/scripts/install @@ -140,16 +140,17 @@ usermod -d $data_path $app 2>&1 #================================================= ynh_script_progression --message="Initializing backend Python virtualenv..." --weight=60 -pushd $final_path/backend || ynh_die - chown -R $app:$app $final_path/backend - sudo -u $app python3 -m venv $final_path/backend/venv - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U wheel pip setuptools - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --requirement $final_path/backend/requirements.txt - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --requirement $final_path/backend/requirements-ynh.txt - sudo -u $app $final_path/backend/venv/bin/python -m spacy download en_core_web_sm - chown -R root:root $final_path/backend +backend_path=$final_path/backend +pushd $backend_path || ynh_die + chown -R $app:$app $backend_path + sudo -u $app python3 -m venv $backend_path/venv + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 + sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 + chown -R root:root $backend_path popd || ynh_die #================================================= @@ -157,13 +158,14 @@ popd || ynh_die #================================================= ynh_script_progression --message="Setting up frontend..." --weight=5 +frontend_path=$final_path/frontend pushd $final_path/frontend || ynh_die - npm install npm@latest --quiet 2>&1 - npx npm cache verify --quiet 2>&1 - npx npm install --quiet 2>&1 - npx npm install d3-zoom d3-force --quiet 2>&1 - SKIP_PREFLIGHT_CHECK=true npx npm run build --quiet 2>&1 - npx npm install serve --quiet 2>&1 + chown -R $app:$app $frontend_path + sudo -u $app touch $frontend_path/.yarnrc + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 + chown -R root:root $frontend_path popd || ynh_die #================================================= @@ -186,19 +188,20 @@ secret_key=$(ynh_string_random -l 64) ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" for file in $final_path/bin/*; do + ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file + echo "$(uniq $file)" > $file ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r $port -f $file - ynh_replace_string -m 8001 -r $backend_port -f $file - ynh_replace_string -m 8002 -r $similarity_port -f $file + ynh_replace_string -m 3000 -r '$httpPort' -f $file + ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file + ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_replace_string -m "python" -r "$final_path/backend/venv/bin/python" -f $file - ynh_replace_string -m "gunicorn" -r "$final_path/backend/venv/bin/gunicorn" -f $file - ynh_replace_string -m "$final_path/backend/venv/bin/$final_path/backend/venv/bin/python" -r "$final_path/backend/venv/bin/python" -f $file - ynh_replace_string -m "$final_path/backend/venv/bin/$final_path/backend/venv/bin/gunicorn" -r "$final_path/backend/venv/bin/gunicorn" -f $file - ynh_store_file_checksum --file="$file" + ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file + ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file done #================================================= @@ -208,8 +211,9 @@ ynh_script_progression --message="Finalizing database..." --weight=1 pushd $final_path || ynh_die chown -R $app:$app $final_path - sudo -u $app $final_path/bin/librephotos-upgrade - sudo -u $app $final_path/bin/librephotos-createadmin "$admin" "$(ynh_user_get_info $admin 'mail')" $(ynh_string_random -l 64) + chown -R $app:$app /var/log/$app + sudo -u $app $final_path/bin/librephotos-upgrade 2>&1 + sudo -u $app $final_path/bin/librephotos-createadmin "$admin" "$(ynh_user_get_info $admin 'mail')" $(ynh_string_random -l 64) 2>&1 for username in $(ynh_user_list); do user_email=$(ynh_user_get_info --username="$username" --key=mail) sudo -u $app $final_path/bin/librephotos-cli shell <<< " @@ -221,7 +225,7 @@ try: user.save() except User.DoesNotExist: User.objects.create_user('$username', email='$user_email', scan_directory='/home/yunohost.multimedia/$username/Picture') -" +" 2>&1 done chown -R root:root $final_path popd || ynh_die diff --git a/scripts/upgrade b/scripts/upgrade index 6715d65..8eaff83 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -125,16 +125,17 @@ usermod -d $data_path $app 2>&1 #================================================= ynh_script_progression --message="Upgrading backend Python virtualenv..." --weight=50 -pushd $final_path/backend || ynh_die - chown -R $app:$app $final_path/backend - sudo -u $app python3 -m venv $final_path/backend/venv - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U wheel pip setuptools - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --requirement $final_path/backend/requirements.txt - sudo -u $app $final_path/backend/venv/bin/pip --cache-dir $final_path/backend/.cache/pip install -U --requirement $final_path/backend/requirements-ynh.txt - sudo -u $app $final_path/backend/venv/bin/python -m spacy download en_core_web_sm - chown -R root:root $final_path/backend +backend_path=$final_path/backend +pushd $backend_path || ynh_die + chown -R $app:$app $backend_path + sudo -u $app python3 -m venv $backend_path/venv + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 + sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 + chown -R root:root $backend_path popd || ynh_die #================================================= @@ -142,13 +143,14 @@ popd || ynh_die #================================================= ynh_script_progression --message="Setting up frontend..." --weight=5 +frontend_path=$final_path/frontend pushd $final_path/frontend || ynh_die - npm install npm@latest --quiet 2>&1 - npx npm cache verify --quiet 2>&1 - npx npm install --quiet 2>&1 - npx npm install d3-zoom d3-force --quiet 2>&1 - SKIP_PREFLIGHT_CHECK=true npx npm run build --quiet 2>&1 - npx npm install serve --quiet 2>&1 + chown -R $app:$app $frontend_path + sudo -u $app touch $frontend_path/.yarnrc + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 + sudo -u $app yarnpkg --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 + chown -R root:root $frontend_path popd || ynh_die #================================================= @@ -171,19 +173,20 @@ secret_key=$(ynh_string_random -l 64) ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" for file in $final_path/bin/*; do + ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file + echo "$(uniq $file)" > $file ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r $port -f $file - ynh_replace_string -m 8001 -r $backend_port -f $file - ynh_replace_string -m 8002 -r $similarity_port -f $file + ynh_replace_string -m 3000 -r '$httpPort' -f $file + ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file + ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_replace_string -m "python" -r "$final_path/backend/venv/bin/python" -f $file - ynh_replace_string -m "gunicorn" -r "$final_path/backend/venv/bin/gunicorn" -f $file - ynh_replace_string -m "$final_path/backend/venv/bin/$final_path/backend/venv/bin/python" -r "$final_path/backend/venv/bin/python" -f $file - ynh_replace_string -m "$final_path/backend/venv/bin/$final_path/backend/venv/bin/gunicorn" -r "$final_path/backend/venv/bin/gunicorn" -f $file - ynh_store_file_checksum --file="$file" + ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file + ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file done #================================================= @@ -193,7 +196,8 @@ ynh_script_progression --message="Finalizing database..." --weight=1 pushd $final_path || ynh_die chown -R $app:$app $final_path - sudo -u $app $final_path/bin/librephotos-upgrade + chown -R $app:$app /var/log/$app + sudo -u $app $final_path/bin/librephotos-upgrade 2>&1 chown -R root:root $final_path popd || ynh_die