mirror of
https://github.com/YunoHost-Apps/librephotos_ynh.git
synced 2024-09-03 19:36:12 +02:00
177 lines
8 KiB
Bash
177 lines
8 KiB
Bash
#!/bin/bash
|
|
|
|
#=================================================
|
|
# COMMON VARIABLES
|
|
#=================================================
|
|
|
|
# dependencies used by the app
|
|
#REMOVEME? pkg_dependencies="libtinfo5 unzip ca-certificates swig libpq-dev postgresql postgresql-contrib postgresql-common ffmpeg libimage-exiftool-perl curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev"
|
|
|
|
arch="$(dpkg --print-architecture)"
|
|
arm64_test=0
|
|
|
|
if ! (apt-cache -q=0 show ufraw-batch |& grep ': No packages found' &>/dev/null); then
|
|
#REMOVEME? pkg_dependencies="$pkg_dependencies ufraw-batch"
|
|
fi
|
|
|
|
#=================================================
|
|
# PERSONAL HELPERS
|
|
#=================================================
|
|
|
|
function unpack_source {
|
|
#REMOVEME? ynh_secure_remove "$install_dir"
|
|
mkdir -p "$install_dir/data_models/"{places365,im2txt}
|
|
ynh_setup_source --source_id="places365" --dest_dir="$install_dir/data_models/places365/"
|
|
ynh_setup_source --source_id="im2txt" --dest_dir="$install_dir/data_models/im2txt/"
|
|
mkdir -p "$data_path"
|
|
ln -sf "$install_dir/data_models" "$data_path/data_models"
|
|
mkdir -p "$data_path/protected_media/"{thumbnails_big,square_thumbnails,square_thumbnails_small,faces}
|
|
mkdir -p "$data_path/data/nextcloud_media"
|
|
mkdir -p "$data_path/matplotlib"
|
|
|
|
mkdir -p ~/.cache/torch/hub/checkpoints/
|
|
ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/"
|
|
|
|
ynh_setup_source --source_id="backend" --dest_dir="$install_dir/backend/"
|
|
ynh_setup_source --source_id="frontend" --dest_dir="$install_dir/frontend/"
|
|
ynh_setup_source --source_id="dlib" --dest_dir="$install_dir/backend/dlib/"
|
|
if [ "$arch" = "arm64" ] || [ "$arm64_test" -eq 1 ]; then
|
|
export CONDA_DIR="$install_dir/backend/conda"
|
|
mkdir -p "$CONDA_DIR"
|
|
if [ "$arch" = "arm64" ]; then
|
|
ynh_setup_source --source_id="miniforge3" --dest_dir="$CONDA_DIR"
|
|
ynh_setup_source --source_id="cmake" --dest_dir="$install_dir/backend/cmake/"
|
|
else
|
|
wget -O "${CONDA_DIR}/Miniforge3-4.10.1-4-Linux-aarch64.sh" https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh 2>&1
|
|
ynh_setup_source --source_id="cmake_amd64" --dest_dir="$install_dir/backend/cmake/"
|
|
fi
|
|
#ynh_setup_source --source_id="faiss" --dest_dir="$install_dir/backend/faiss/"
|
|
fi
|
|
|
|
mkdir -p "/var/log/$app"
|
|
}
|
|
|
|
function set_up_backend {
|
|
backend_path="$install_dir/backend"
|
|
pushd "$backend_path"
|
|
chown -R $app:$app "$backend_path"
|
|
sudo -u $app python3 -m venv $backend_path/venv
|
|
path_prefix="$backend_path/venv/bin"
|
|
if [ "$arch" = "arm64" ] || [ "$arm64_test" -eq 1 ]; then
|
|
path_prefix="$backend_path/cmake/bin:$CONDA_DIR/condabin:$CONDA_DIR/bin:$path_prefix"
|
|
fi
|
|
local python_path="$path_prefix:$(sudo -u $app bash -c 'echo $PATH')"
|
|
local cache_dir="$backend_path/.cache/pip"
|
|
sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U wheel pip setuptools 2>&1
|
|
if [ "$arch" = "arm64" ] || [ "$arm64_test" -eq 1 ]; then
|
|
sudo -u $app env "CONDA_DIR=$CONDA_DIR" bash "${CONDA_DIR}/Miniforge3-4.10.1-4-Linux-aarch64.sh" -bu -p "${CONDA_DIR}"
|
|
sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U torch==1.8.1 torchvision==0.9.1 -f https://torch.maku.ml/whl/stable.html 2>&1
|
|
sudo -u $app env "PATH=$python_path" conda install -y numpy psycopg2 cython pandas scikit-learn=0.24.1 scikit-image=0.18.1 spacy=2.3.5 gevent=20.12.1 matplotlib=3.3.2 faiss-cpu==1.7.0
|
|
#pushd "$backend_path/faiss"
|
|
# sudo -u $app env "PATH=$python_path" cmake -B build . -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DFAISS_OPT_LEVEL=generic
|
|
# sudo -u $app env "PATH=$python_path" make -C build -j faiss
|
|
# sudo -u $app env "PATH=$python_path" make -C build -j swigfaiss
|
|
# cd "build/faiss/python"
|
|
# sudo -u $app env "PATH=$python_path" python setup.py install
|
|
#popd
|
|
sed -i "/spacy==2.3.2/d" "$backend_path/requirements.txt"
|
|
sed -i "/sklearn==0.0/d" "$backend_path/requirements.txt"
|
|
sed -i "/gevent==20.9.0/d" "$backend_path/requirements.txt"
|
|
sed -i "/scipy==1.5.3/d" "$backend_path/requirements.txt"
|
|
sed -i "s/pytz==2020.1/pytz>=2021.1/" "$backend_path/requirements.txt"
|
|
sed -i "s/Pillow==8.1.0/Pillow>=8.1.2/" "$backend_path/requirements.txt"
|
|
sed -i "/faiss-cpu==1.7.0/d" "$backend_path/requirements.txt"
|
|
else
|
|
sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U torch==1.8.0+cpu torchvision==0.9.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1
|
|
fi
|
|
pushd "$backend_path/dlib"
|
|
sudo -u $app env "PATH=$python_path" python setup.py install 2>&1
|
|
popd
|
|
sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements.txt" 2>&1
|
|
sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements-ynh.txt" 2>&1
|
|
#if [ "$arch" = "arm64" ] || [ "$arm64_test" -eq 1 ]; then
|
|
#sudo -u $app unzip "$CONDA_DIR/lib/python3.8/site-packages/"faiss*.egg -d "$CONDA_DIR/lib/python3.8/site-packages/"
|
|
#fi
|
|
chown -R root:root "$backend_path"
|
|
popd
|
|
}
|
|
|
|
function install_dlib {
|
|
pushd "$backend_path/dlib"
|
|
sudo -u $app "$backend_path/venv/bin/python" setup.py install 2>&1
|
|
popd
|
|
}
|
|
|
|
function set_node_vars {
|
|
ynh_exec_warn_less ynh_install_nodejs --nodejs_version=13
|
|
ynh_use_nodejs
|
|
node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH')
|
|
|
|
}
|
|
|
|
function set_up_frontend {
|
|
set_node_vars
|
|
frontend_path=$install_dir/frontend
|
|
pushd $install_dir/frontend
|
|
chown -R $app:$app $frontend_path
|
|
sudo -u $app touch $frontend_path/.yarnrc
|
|
sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1
|
|
sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1
|
|
sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1
|
|
chown -R root:root $frontend_path
|
|
popd
|
|
}
|
|
|
|
function add_configuations {
|
|
#REMOVEME? secret_key=$(ynh_app_setting_get --app=$app --key=secret_key)
|
|
|
|
if [ -z $secret_key ]; then
|
|
secret_key=$(ynh_string_random -l 64)
|
|
ynh_app_setting_set --app=$app --key=secret_key --value=$secret_key
|
|
fi
|
|
|
|
ynh_add_config --template="librephotos.env" --destination="$install_dir/librephotos.env"
|
|
}
|
|
|
|
function upgrade_db {
|
|
pushd "$install_dir/backend"
|
|
chown -R $app:$app "$install_dir/backend"
|
|
chown -R $app:$app "/var/log/$app"
|
|
sudo -u $app bash -c "
|
|
set -a
|
|
export PATH=\"$path_prefix:"'$PATH'"\"
|
|
source \"$install_dir\"/librephotos.env
|
|
python3 manage.py showmigrations
|
|
python3 manage.py migrate
|
|
python3 manage.py showmigrations
|
|
" 2>&1
|
|
popd
|
|
set_permissions
|
|
}
|
|
|
|
function set_permissions {
|
|
chown -R root:$app "$install_dir"
|
|
chmod -R g=u,g-w,o-rwx "$install_dir"
|
|
chown -R $app:$app "$data_path"
|
|
chmod -R g=u,g-w,o-rwx "$data_path"
|
|
chown -R $app:$app "$install_dir/data_models"
|
|
chown -R $app:$app "/var/log/$app"
|
|
chmod -R g-w,o-rwx "/var/log/$app"
|
|
setfacl -n -m user:www-data:rx "$data_path"
|
|
setfacl -nR -m u:www-data:rx -m d:u:www-data:rx "$data_path/protected_media" "$data_path/data" "$data_path/data/nextcloud_media"
|
|
}
|
|
|
|
function set_up_logrotate {
|
|
ynh_use_logrotate --logfile="/var/log/$app/command_build_similarity_index.log" --specific_user="$app/$app" --non_append
|
|
ynh_use_logrotate --logfile="/var/log/$app/gunicorn_django.log" --specific_user="$app/$app"
|
|
ynh_use_logrotate --logfile="/var/log/$app/image_similarity.log" --specific_user="$app/$app"
|
|
ynh_use_logrotate --logfile="/var/log/$app/ownphotos.log" --specific_user="$app/$app"
|
|
}
|
|
|
|
#=================================================
|
|
# EXPERIMENTAL HELPERS
|
|
#=================================================
|
|
|
|
#=================================================
|
|
# FUTURE OFFICIAL HELPERS
|
|
#=================================================
|