diff --git a/scripts/_common.sh b/scripts/_common.sh index c414b6e..6179262 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -11,6 +11,105 @@ pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-co # PERSONAL HELPERS #================================================= +function unpack_source { + mkdir -p $final_path/data_models/{places365,im2txt} + ynh_setup_source --source_id="places365_model" --dest_dir="$final_path/data_models/places365/model/" + ynh_setup_source --source_id="im2txt_model" --dest_dir="$final_path/data_models/im2txt/model/" + ynh_setup_source --source_id="im2txt_data" --dest_dir="$final_path/data_models/im2txt/data/" + mkdir -p $data_path + ln -sf "$final_path/data_models" "$data_path/data_models" + mkdir -p $data_path/protected_media/{thumbnails_big,square_thumbnails,square_thumbnails_small,faces} + mkdir -p $data_path/data/nextcloud_media + mkdir -p $data_path/matplotlib + + mkdir -p ~/.cache/torch/hub/checkpoints/ + ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/" + + ynh_setup_source --source_id="backend" --dest_dir="$final_path/backend/" + ynh_setup_source --source_id="frontend" --dest_dir="$final_path/frontend/" + ynh_setup_source --source_id="linux" --dest_dir="$final_path/linux/" + mkdir -p "$final_path/bin" + mv -f "$final_path/linux/ressources/bin/"* "$final_path/bin" + ynh_secure_remove --file="$final_path/linux" + + mkdir -p /var/log/$app +} + +function set_up_virtualenv { + backend_path=$final_path/backend + pushd $backend_path || ynh_die + chown -R $app:$app $backend_path + sudo -u $app python3 -m venv $backend_path/venv + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 + sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 + sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 + chown -R root:root $backend_path + popd || ynh_die +} + +function set_up_frontend { + ynh_exec_warn_less ynh_install_nodejs --nodejs_version=10 + ynh_use_nodejs + node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH') + + frontend_path=$final_path/frontend + pushd $final_path/frontend || ynh_die + chown -R $app:$app $frontend_path + sudo -u $app touch $frontend_path/.yarnrc + sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 + sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 + sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 + chown -R root:root $frontend_path + popd || ynh_die +} + +function add_configuations { + secret_key=$(ynh_app_setting_get --app=$app --key=secret_key) + + if [ -z $secret_key ]; then + secret_key=$(ynh_string_random -l 64) + ynh_app_setting_set --app=$app --key=secret_key --value=$secret_key + fi + + ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" + + for file in $final_path/bin/*; do + ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file + echo "$(uniq $file)" > $file + ynh_replace_string -m "source $final_path/librephotos.env" -r "source $final_path/librephotos.env\nexport PATH=\$NODEJS_PATH:\$PATH" -f $file + echo "$(uniq $file)" > $file + ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file + ynh_replace_string -m 3000 -r '$httpPort' -f $file + ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file + ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file + ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file + ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file + ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file + ynh_replace_special_string -m 'EOF' -r '' -f $file + ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file + ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file + ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file + ynh_replace_string -m 'npm install' -r " " -f $file + ynh_replace_string -m 'npm' -r "yarn" -f $file + done +} + +function set_permissions { + chown -R root:$app $final_path + chmod -R g=u,g-w,o-rwx $final_path + chown -R $app:$app $data_path + chmod -R g=u,g-w,o-rwx $data_path + chown -R $app:$app $final_path/data_models + chown -R $app:$app /var/log/$app + chmod -R g-w,o-rwx /var/log/$app + setfacl -n -m user:www-data:rx $data_path + setfacl -n -R -m user:www-data:rx -m default:user:www-data:rx $data_path/protected_media $data_path/data $data_path/data/nextcloud_media +} + #================================================= # EXPERIMENTAL HELPERS #================================================= @@ -23,24 +122,24 @@ pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-co # # usage: ynh_multimedia_build_main_dir ynh_multimedia_build_main_dir () { - local ynh_media_release="v1.2" - local checksum="806a827ba1902d6911095602a9221181" + local ynh_media_release="v1.2" + local checksum="806a827ba1902d6911095602a9221181" - # Download yunohost.multimedia scripts - wget -nv https://github.com/YunoHost-Apps/yunohost.multimedia/archive/${ynh_media_release}.tar.gz 2>&1 + # Download yunohost.multimedia scripts + wget -nv https://github.com/YunoHost-Apps/yunohost.multimedia/archive/${ynh_media_release}.tar.gz 2>&1 - # Check the control sum - echo "${checksum} ${ynh_media_release}.tar.gz" | md5sum -c --status \ - || ynh_die "Corrupt source" + # Check the control sum + echo "${checksum} ${ynh_media_release}.tar.gz" | md5sum -c --status \ + || ynh_die "Corrupt source" - # Check if the package acl is installed. Or install it. - ynh_package_is_installed 'acl' \ - || ynh_package_install acl + # Check if the package acl is installed. Or install it. + ynh_package_is_installed 'acl' \ + || ynh_package_install acl - # Extract - mkdir yunohost.multimedia-master - tar -xf ${ynh_media_release}.tar.gz -C yunohost.multimedia-master --strip-components 1 - ./yunohost.multimedia-master/script/ynh_media_build.sh + # Extract + mkdir yunohost.multimedia-master + tar -xf ${ynh_media_release}.tar.gz -C yunohost.multimedia-master --strip-components 1 + ./yunohost.multimedia-master/script/ynh_media_build.sh } # Grant write access to multimedia directories to a specified user @@ -49,7 +148,7 @@ ynh_multimedia_build_main_dir () { # # | arg: user_name - User to be granted write access ynh_multimedia_addaccess () { - local user_name=$1 - groupadd -f multimedia - usermod -a -G multimedia $user_name + local user_name=$1 + groupadd -f multimedia + usermod -a -G multimedia $user_name } diff --git a/scripts/install b/scripts/install index 0628d33..bd51dde 100755 --- a/scripts/install +++ b/scripts/install @@ -92,30 +92,7 @@ ynh_psql_setup_db --db_user=$db_user --db_name=$db_name #================================================= ynh_script_progression --message="Setting up source files..." --weight=5 -ynh_app_setting_set --app=$app --key=final_path --value=$final_path -ynh_app_setting_set --app=$app --key=data_path --value=$data_path - -mkdir -p $final_path/data_models/{places365,im2txt} -ynh_setup_source --source_id="places365_model" --dest_dir="$final_path/data_models/places365/model/" -ynh_setup_source --source_id="im2txt_model" --dest_dir="$final_path/data_models/im2txt/model/" -ynh_setup_source --source_id="im2txt_data" --dest_dir="$final_path/data_models/im2txt/data/" -mkdir -p $data_path -ln -sf "$final_path/data_models" "$data_path/data_models" -mkdir -p $data_path/protected_media/{thumbnails_big,square_thumbnails,square_thumbnails_small,faces} -mkdir -p $data_path/data/nextcloud_media -mkdir -p $data_path/matplotlib - -mkdir -p ~/.cache/torch/hub/checkpoints/ -ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/" - -ynh_setup_source --source_id="backend" --dest_dir="$final_path/backend/" -ynh_setup_source --source_id="frontend" --dest_dir="$final_path/frontend/" -ynh_setup_source --source_id="linux" --dest_dir="$final_path/linux/" -mkdir -p "$final_path/bin" -mv -f "$final_path/linux/ressources/bin/"* "$final_path/bin" -ynh_secure_remove --file="$final_path/linux" - -mkdir -p /var/log/$app +unpack_source #================================================= # NGINX CONFIGURATION @@ -131,8 +108,7 @@ ynh_add_nginx_config backend_port ynh_script_progression --message="Configuring system user..." --weight=1 # Create a system user -ynh_system_user_create --username=$app -usermod -d $data_path $app 2>&1 +ynh_system_user_create --username=$app --home_dir=$data_path #================================================= # SPECIFIC SETUP @@ -141,37 +117,14 @@ usermod -d $data_path $app 2>&1 #================================================= ynh_script_progression --message="Initializing backend Python virtualenv..." --weight=60 -backend_path=$final_path/backend -pushd $backend_path || ynh_die - chown -R $app:$app $backend_path - sudo -u $app python3 -m venv $backend_path/venv - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 - sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 - chown -R root:root $backend_path -popd || ynh_die +set_up_virtualenv #================================================= # SET UP FRONTEND #================================================= ynh_script_progression --message="Setting up frontend..." --weight=15 -ynh_exec_warn_less ynh_install_nodejs --nodejs_version=10 -ynh_use_nodejs -node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH') - -frontend_path=$final_path/frontend -pushd $final_path/frontend || ynh_die - chown -R $app:$app $frontend_path - sudo -u $app touch $frontend_path/.yarnrc - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 - chown -R root:root $frontend_path -popd || ynh_die +set_up_frontend #================================================= # SETUP SYSTEMD @@ -189,29 +142,7 @@ ynh_add_systemd_config --service=$app-worker --template=worker.service #================================================= ynh_script_progression --message="Generating configuration files..." --weight=1 -secret_key=$(ynh_string_random -l 64) -ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" - -for file in $final_path/bin/*; do - ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file - echo "$(uniq $file)" > $file - ynh_replace_string -m "source $final_path/librephotos.env" -r "source $final_path/librephotos.env\nexport PATH=\$NODEJS_PATH:\$PATH" -f $file - echo "$(uniq $file)" > $file - ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r '$httpPort' -f $file - ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file - ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file - ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file - ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file - ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file - ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file - ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file - ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file - ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file - ynh_replace_string -m 'npm install' -r " " -f $file - ynh_replace_string -m 'npm' -r "yarn" -f $file -done +add_configuations #================================================= # FINALIZE DATABASE @@ -257,15 +188,7 @@ fi #================================================= # Set permissions to app files -chown -R root:$app $final_path -chmod -R g=u,g-w,o-rwx $final_path -chown -R $app:$app $data_path -chmod -R g=u,g-w,o-rwx $data_path -chown -R $app:$app $final_path/data_models -chown -R $app:$app /var/log/$app -chmod -R g-w,o-rwx /var/log/$app -setfacl -n -m user:www-data:rx $data_path -setfacl -n -R -m user:www-data:rx -m default:user:www-data:rx $data_path/protected_media $data_path/data $data_path/data/nextcloud_media +set_permissions #================================================= # SETUP LOGROTATE diff --git a/scripts/restore b/scripts/restore index 483ef7e..f1f28a9 100755 --- a/scripts/restore +++ b/scripts/restore @@ -85,8 +85,7 @@ ln -s $final_path/data_models $data_path/data_models ynh_script_progression --message="Recreating the dedicated system user..." --weight=1 # Create the dedicated user (if not existing) -ynh_system_user_create --username=$app -usermod -d $data_path $app 2>&1 +ynh_system_user_create --username=$app --home_dir=$data_path #================================================= # RESTORE VARIOUS FILES diff --git a/scripts/upgrade b/scripts/upgrade index f908a18..5cc2121 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -68,30 +68,7 @@ ynh_systemd_action --service_name=$app-worker --action="stop" --log_path="/var/l if [ "$upgrade_type" == "UPGRADE_APP" ] then - ynh_script_progression --message="Upgrading source files..." --weight=1 - - # Download, check integrity, uncompress and patch the sources - mkdir -p $final_path/data_models/{places365,im2txt} - ynh_setup_source --source_id="places365_model" --dest_dir="$final_path/data_models/places365/model/" - ynh_setup_source --source_id="im2txt_model" --dest_dir="$final_path/data_models/im2txt/model/" - ynh_setup_source --source_id="im2txt_data" --dest_dir="$final_path/data_models/im2txt/data/" - mkdir -p $data_path - ln -sf "$final_path/data_models" "$data_path/data_models" - mkdir -p $data_path/protected_media/{thumbnails_big,square_thumbnails,square_thumbnails_small,faces} - mkdir -p $data_path/data/nextcloud_media - mkdir -p $data_path/matplotlib - - mkdir -p ~/.cache/torch/hub/checkpoints/ - ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/" - - ynh_setup_source --source_id="backend" --dest_dir="$final_path/backend/" - ynh_setup_source --source_id="frontend" --dest_dir="$final_path/frontend/" - ynh_setup_source --source_id="linux" --dest_dir="$final_path/linux/" - mkdir -p "$final_path/bin" - mv -f "$final_path/linux/ressources/bin/"* "$final_path/bin" - ynh_secure_remove --file="$final_path/linux" - - mkdir -p /var/log/$app + unpack_source fi #================================================= @@ -116,47 +93,23 @@ ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ st ynh_script_progression --message="Making sure dedicated system user exists..." --weight=1 # Create a dedicated user (if not existing) -ynh_system_user_create --username=$app -usermod -d $data_path $app 2>&1 +ynh_system_user_create --username=$app --home_dir=$data_path #================================================= # SPECIFIC UPGRADE #================================================= # SET UP VIRTUALENV #================================================= -ynh_script_progression --message="Upgrading backend Python virtualenv..." --weight=50 +ynh_script_progression --message="Upgradingynh_app_setting_setbackend Python virtualenv..." --weight=50 -backend_path=$final_path/backend -pushd $backend_path || ynh_die - chown -R $app:$app $backend_path - sudo -u $app python3 -m venv $backend_path/venv - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 - sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 - chown -R root:root $backend_path -popd || ynh_die +set_up_virtualenv #================================================= # SET UP FRONTEND #================================================= ynh_script_progression --message="Setting up frontend..." --weight=5 -ynh_exec_warn_less ynh_install_nodejs --nodejs_version=10 -ynh_use_nodejs -node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH') - -frontend_path=$final_path/frontend -pushd $final_path/frontend || ynh_die - chown -R $app:$app $frontend_path - sudo -u $app touch $frontend_path/.yarnrc - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 - sudo -u $app PATH=$node_PATH yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 - chown -R root:root $frontend_path -popd || ynh_die +set_up_frontend #================================================= # SETUP SYSTEMD @@ -174,29 +127,7 @@ ynh_add_systemd_config --service=$app-worker --template=worker.service #================================================= ynh_script_progression --message="Generating configuration files..." --weight=1 -secret_key=$(ynh_string_random -l 64) -ynh_add_config --template="librephotos.env" --destination="$final_path/librephotos.env" - -for file in $final_path/bin/*; do - ynh_replace_string -m '#!/usr/bin/env bash' -r "#!/usr/bin/env bash\nsource $final_path/librephotos.env" -f $file - echo "$(uniq $file)" > $file - ynh_replace_string -m "source $final_path/librephotos.env" -r "source $final_path/librephotos.env\nexport PATH=\$NODEJS_PATH:\$PATH" -f $file - echo "$(uniq $file)" > $file - ynh_replace_string -m "/usr/lib/librephotos" -r "$final_path" -f $file - ynh_replace_string -m 3000 -r '$httpPort' -f $file - ynh_replace_string -m 8001 -r '$BACKEND_PORT' -f $file - ynh_replace_string -m 8002 -r '$IMAGE_SIMILARITY_SERVER_PORT' -f $file - ynh_replace_string -m "/etc/librephotos" -r $final_path -f $file - ynh_replace_string -m "librephotos-backend.env" -r "librephotos.env" -f $file - ynh_replace_special_string -m 'su - -s $(which bash) librephotos << EOF' -r '' -f $file - ynh_replace_special_string -m 'EOF' -r '' -f $file - ynh_replace_string -m "python3" -r "$backend_path/venv/bin/python3" -f $file - ynh_replace_string -m "gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file - ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/python" -r "$backend_path/venv/bin/python" -f $file - ynh_replace_string -m "$backend_path/venv/bin/$backend_path/venv/bin/gunicorn --workers" -r "$backend_path/venv/bin/gunicorn --workers" -f $file - ynh_replace_string -m 'npm install' -r " " -f $file - ynh_replace_string -m 'npm' -r "yarn" -f $file -done +add_configuations #================================================= # FINALIZE DATABASE @@ -228,15 +159,7 @@ fi #================================================= # Set permissions to app files -chown -R root:$app $final_path -chmod -R g=u,g-w,o-rwx $final_path -chown -R $app:$app $data_path -chmod -R g=u,g-w,o-rwx $data_path -chown -R $app:$app $final_path/data_models -chown -R $app:$app /var/log/$app -chmod -R g-w,o-rwx /var/log/$app -setfacl -n -m user:www-data:rx $data_path -setfacl -n -R -m user:www-data:rx -m default:user:www-data:rx $data_path/protected_media $data_path/data $data_path/data/nextcloud_media +set_permissions #================================================= # SETUP LOGROTATE