diff --git a/README.md b/README.md index 3eda5dd..90970e8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in ## Overview LibrePhotos is a Google Photos-like app. It supports automatic classification of faces, grouping photos by date, location, or objects present, and album creation. -**Shipped version:** 2021-05-25 +**Shipped version:** 2021-05-31 ## Screenshots diff --git a/README_fr.md b/README_fr.md index 79a9b4b..27f23cc 100644 --- a/README_fr.md +++ b/README_fr.md @@ -11,7 +11,7 @@ Si vous n'avez pas YunoHost, consultez [le guide](https://yunohost.org/#/install ## Vue d'ensemble LibrePhotos est un sercice inspiré par Google Photos. Il prend en charge la classification des visages, le groupement des photos par date, la localisation géographique, ou objets présents, et la création d'albums. -**Version incluse :** 2021-05-25 +**Version incluse :** 2021-05-31 ## Captures d'écran diff --git a/conf/backend.src b/conf/backend.src index a5ec116..ecdd88c 100644 --- a/conf/backend.src +++ b/conf/backend.src @@ -1,7 +1,7 @@ -SOURCE_URL=https://github.com/LibrePhotos/librephotos/tarball/483a90bc61196364b2d37ea7749e501cb9de50b7 -SOURCE_SUM=1d838a739f16a994b25f4511334f65c5a246b46a746831611fe4296ffdc117fb +SOURCE_URL=https://github.com/LibrePhotos/librephotos/tarball/bd4fabf104ea818a4b8fae61e6e41c08ad89ba16 +SOURCE_SUM=6cd32a2d75173c851aed76b8f045f95b67fafac4be9ff499c498c6ceba843eb2 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=true -SOURCE_FILENAME=483a90bc61196364b2d37ea7749e501cb9de50b7 +SOURCE_FILENAME=bd4fabf104ea818a4b8fae61e6e41c08ad89ba16 SOURCE_EXTRACT=true diff --git a/conf/dlib.src b/conf/dlib.src new file mode 100644 index 0000000..8c2eeea --- /dev/null +++ b/conf/dlib.src @@ -0,0 +1,7 @@ +SOURCE_URL=https://github.com/davisking/dlib/archive/refs/tags/v19.22.tar.gz +SOURCE_SUM=5f44b67f762691b92f3e41dcf9c95dd0f4525b59cacb478094e511fdacb5c096 +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=tar.gz +SOURCE_IN_SUBDIR=true +SOURCE_FILENAME=v19.22.tar.gz +SOURCE_EXTRACT=true diff --git a/conf/faiss.src b/conf/faiss.src new file mode 100644 index 0000000..bff08fb --- /dev/null +++ b/conf/faiss.src @@ -0,0 +1,7 @@ +SOURCE_URL=https://github.com/facebookresearch/faiss/archive/refs/tags/v1.7.1.tar.gz +SOURCE_SUM=d676d3107ad41203a49e0afda2630519299dc8666f8d23322cbe1eac0c431871 +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=tar.gz +SOURCE_IN_SUBDIR=true +SOURCE_FILENAME=v1.7.1.tar.gz +SOURCE_EXTRACT=true diff --git a/conf/frontend.src b/conf/frontend.src index 41d09c1..cf6594f 100644 --- a/conf/frontend.src +++ b/conf/frontend.src @@ -1,7 +1,7 @@ -SOURCE_URL=https://github.com/LibrePhotos/librephotos-frontend/tarball/ab1c9e3fe7157580f440338b5b7b1096c055dea8 -SOURCE_SUM=00df92d233492f6a29c5faf900116fb09646dc099edc268d70a019eb4240e411 +SOURCE_URL=https://github.com/LibrePhotos/librephotos-frontend/tarball/86b2bb61b4a3e43e200d3fe22766b92c73c30e74 +SOURCE_SUM=82ddfd1d698d3d8c4645449ac4c0fc442bae74ddb1a77934ef9516672f4731a6 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=true -SOURCE_FILENAME=ab1c9e3fe7157580f440338b5b7b1096c055dea8 +SOURCE_FILENAME=86b2bb61b4a3e43e200d3fe22766b92c73c30e74 SOURCE_EXTRACT=true diff --git a/conf/linux.src b/conf/linux.src index 3f789b1..187377c 100644 --- a/conf/linux.src +++ b/conf/linux.src @@ -1,7 +1,7 @@ -SOURCE_URL=https://github.com/LibrePhotos/librephotos-linux/tarball/a8b77d684708a268485a46bb5c06d59dffc764df -SOURCE_SUM=5128240de9e543c0c360608528656e35fd828633284457519b6de9cb905b56fa +SOURCE_URL=https://github.com/LibrePhotos/librephotos-linux/tarball/3891c65924dc49d87aea0bc9b79448e9d56cd954 +SOURCE_SUM=1c6b4e9b12f3bea6e298494ad7880483190b4deefb006c9c8d22f840dad2e1ba SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=true -SOURCE_FILENAME=a8b77d684708a268485a46bb5c06d59dffc764df +SOURCE_FILENAME=3891c65924dc49d87aea0bc9b79448e9d56cd954 SOURCE_EXTRACT=true diff --git a/conf/miniforge3.src b/conf/miniforge3.src new file mode 100644 index 0000000..a3fb707 --- /dev/null +++ b/conf/miniforge3.src @@ -0,0 +1,7 @@ +SOURCE_URL=https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh +SOURCE_SUM=68f11be0b8272b9218f62fa3ba1b7c58783427e65db7b7ba77e7cdf91f099540 +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=sh +SOURCE_IN_SUBDIR=false +SOURCE_FILENAME=Miniforge3-4.10.1-4-Linux-aarch64.sh +SOURCE_EXTRACT=false diff --git a/conf/nginx.conf b/conf/nginx.conf index aa561bb..e00205e 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -15,13 +15,24 @@ location ~ ^/(api|media)/ { #unicorn need to listen on [::1] proxy_pass http://127.0.0.1:__BACKEND_PORT__; } + # Django media location /protected_media { internal; alias /home/yunohost.app/__NAME__/protected_media/; } + +location /static/drf-yasg { + proxy_pass http://127.0.0.1:__BACKEND_PORT__; +} + +location /data { + internal; + alias /home/yunohost.app/__NAME__/data/; +} + # Original Photos -location /original { +location /original { internal; alias /home/yunohost.app/__NAME__/data/; } diff --git a/manifest.json b/manifest.json index b28b8a6..d983323 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "A photo viewer and manager similar to Google Photos", "fr": "Un gestionnaire de photos semblable à Google Photos" }, - "version": "0.0.20210525~ynh1", + "version": "0.0.20210531~ynh1", "url": "https://github.com/LibrePhotos/librephotos", "license": "MIT", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 7696b97..ceebce7 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,22 +5,25 @@ #================================================= # dependencies used by the app -pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common ffmpeg libimage-exiftool-perl curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch" +pkg_dependencies="unzip ca-certificates swig libpq-dev postgresql postgresql-contrib postgresql-common ffmpeg libimage-exiftool-perl curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch" + +arch="$(dpkg --print-architecture)" #================================================= # PERSONAL HELPERS #================================================= function unpack_source { - mkdir -p $final_path/data_models/{places365,im2txt} + ynh_secure_remove "$final_path" + mkdir -p "$final_path/data_models/"{places365,im2txt} ynh_setup_source --source_id="places365_model" --dest_dir="$final_path/data_models/places365/model/" ynh_setup_source --source_id="im2txt_model" --dest_dir="$final_path/data_models/im2txt/model/" ynh_setup_source --source_id="im2txt_data" --dest_dir="$final_path/data_models/im2txt/data/" - mkdir -p $data_path + mkdir -p "$data_path" ln -sf "$final_path/data_models" "$data_path/data_models" - mkdir -p $data_path/protected_media/{thumbnails_big,square_thumbnails,square_thumbnails_small,faces} - mkdir -p $data_path/data/nextcloud_media - mkdir -p $data_path/matplotlib + mkdir -p "$data_path/protected_media/"{thumbnails_big,square_thumbnails,square_thumbnails_small,faces} + mkdir -p "$data_path/data/nextcloud_media" + mkdir -p "$data_path/matplotlib" mkdir -p ~/.cache/torch/hub/checkpoints/ ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/" @@ -28,30 +31,72 @@ function unpack_source { ynh_setup_source --source_id="backend" --dest_dir="$final_path/backend/" ynh_setup_source --source_id="frontend" --dest_dir="$final_path/frontend/" ynh_setup_source --source_id="linux" --dest_dir="$final_path/linux/" + ynh_setup_source --source_id="dlib" --dest_dir="$final_path/backend/dlib/" + if [ "$arch" = "aarch64" ]; then + export CONDA_DIR="$final_path/backend/conda" + mkdir -p "$CONDA_DIR" + ynh_setup_source --source_id="miniforge3" --dest_dir="$CONDA_DIR" + ynh_setup_source --source_id="faiss" --dest_dir="$final_path/backend/faiss/" + fi mkdir -p "$final_path/bin" mv -f "$final_path/linux/ressources/bin/"* "$final_path/bin" ynh_secure_remove --file="$final_path/linux" - mkdir -p /var/log/$app + mkdir -p "/var/log/$app" } function set_up_virtualenv { - backend_path=$final_path/backend - pushd $backend_path || ynh_die - chown -R $app:$app $backend_path + backend_path="$final_path/backend" + pushd "$backend_path" + chown -R $app:$app "$backend_path" sudo -u $app python3 -m venv $backend_path/venv - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.8.0+cpu torchvision==0.9.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1 - sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1 - sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1 - chown -R root:root $backend_path - popd || ynh_die + local python_path="$backend_path/venv/bin:$(sudo -u $app bash -c 'echo $PATH')" + local cache_dir="$backend_path/.cache/pip" + sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U wheel pip setuptools 2>&1 + if [ "$arch" = "aarch64" ]; then + sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://torch.maku.ml/whl/stable.html 2>&1 + sudo -u $app env "CONDA_DIR=$CONDA_DIR" bash "${CONDA_DIR}/Miniforge3-4.10.1-4-Linux-aarch64.sh" -b -p "${CONDA_DIR}" + local conda="$CONDA_DIR/condabin/conda" + python_path="$CONDA_DIR/condabin:$CONDA_DIR/bin:$python_path" + sudo -u $app env "PATH=$python_path" conda install -y numpy psycopg2 cython pandas scikit-learn=0.24.1 scikit-image=0.18.1 spacy=2.3.5 gevent=20.12.1 matplotlib=3.3.2 + pushd "$backend_path/faiss" + sudo -u $app cmake -B build . + sudo -u $app make -C build -j faiss + sudo -u $app make -C build -j swigfaiss + cd "build/faiss/python" + sudo -u $app env "PATH=$python_path" python setup.py install + popd + sed -i "/spacy==2.3.2/d" "$backend_path/requirements.txt" + sed -i "/sklearn==0.0/d" "$backend_path/requirements.txt" + sed -i "/gevent==20.9.0/d" "$backend_path/requirements.txt" + sed -i "/scipy==1.5.3/d" "$backend_path/requirements.txt" + sed -i "s/pytz==2020.1/pytz>=2021.1/" "$backend_path/requirements.txt" + sed -i "s/Pillow==8.1.0/Pillow>=8.1.2/" "$backend_path/requirements.txt" + sed -i "/faiss-cpu==1.7.0/d" "$backend_path/requirements.txt" + else + sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U torch==1.8.0+cpu torchvision==0.9.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1 + fi + pushd "$backend_path/dlib" + sudo -u $app env "PATH=$python_path" python setup.py install 2>&1 + popd + sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements.txt" 2>&1 + sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements-ynh.txt" 2>&1 + sudo -u $app env "PATH=$python_path" python -m spacy download en_core_web_sm 2>&1 + if [ "$arch" = "aarch64" ]; then + unzip "$CONDA_DIR/lib/python3.8/site-packages/"faiss*.egg -d "$CONDA_DIR/lib/python3.8/site-packages/" + fi + chown -R root:root "$backend_path" + popd +} + +function install_dlib { + pushd "$backend_path/dlib" + sudo -u $app "$backend_path/venv/bin/python" setup.py install 2>&1 + popd } function set_node_vars { - ynh_exec_warn_less ynh_install_nodejs --nodejs_version=10 + ynh_exec_warn_less ynh_install_nodejs --nodejs_version=13 ynh_use_nodejs node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH') @@ -60,14 +105,14 @@ function set_node_vars { function set_up_frontend { set_node_vars frontend_path=$final_path/frontend - pushd $final_path/frontend || ynh_die + pushd $final_path/frontend chown -R $app:$app $frontend_path sudo -u $app touch $frontend_path/.yarnrc sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1 sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1 sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1 chown -R root:root $frontend_path - popd || ynh_die + popd } function add_configuations { @@ -103,15 +148,15 @@ function add_configuations { } function set_permissions { - chown -R root:$app $final_path - chmod -R g=u,g-w,o-rwx $final_path - chown -R $app:$app $data_path - chmod -R g=u,g-w,o-rwx $data_path - chown -R $app:$app $final_path/data_models - chown -R $app:$app /var/log/$app - chmod -R g-w,o-rwx /var/log/$app - setfacl -n -m user:www-data:rx $data_path - setfacl -n -R -m user:www-data:rx -m default:user:www-data:rx $data_path/protected_media $data_path/data $data_path/data/nextcloud_media + chown -R root:$app "$final_path" + chmod -R g=u,g-w,o-rwx "$final_path" + chown -R $app:$app "$data_path" + chmod -R g=u,g-w,o-rwx "$data_path" + chown -R $app:$app "$final_path/data_models" + chown -R $app:$app "/var/log/$app" + chmod -R g-w,o-rwx "/var/log/$app" + setfacl -n -m user:www-data:rx "$data_path" + setfacl -nR -m u:www-data:rx -m d:u:www-data:rx "$data_path/protected_media" "$data_path/data" "$data_path/data/nextcloud_media" } function set_up_logrotate { diff --git a/scripts/upgrade b/scripts/upgrade index f31f815..fb99640 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -62,6 +62,17 @@ ynh_systemd_action --service_name=$app-frontend --action="stop" --log_path="syst ynh_systemd_action --service_name=$app-image-similarity --action="stop" --log_path="/var/log/$app/image_similarity.log" ynh_systemd_action --service_name=$app-worker --action="stop" --log_path="/var/log/$app/$app-worker.log" +#================================================= +# ENSURE DOWNWARD COMPATIBILITY +#================================================= +ynh_script_progression --message="Ensuring downward compatibility..." --time --weight=1 + +# Remove old node version +ynh_use_nodejs +if [ "$nodejs_version" -eq 10 ]; then + ynh_remove_nodejs +fi + #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE #=================================================