Update to 2021-05-31

README.md

@@ -11,7 +11,7 @@ If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/in
 ## Overview
 LibrePhotos is a Google Photos-like app. It supports automatic classification of faces, grouping photos by date, location, or objects present, and album creation.
 
-**Shipped version:** 2021-05-25
+**Shipped version:** 2021-05-31
 
 ## Screenshots
 

README_fr.md

@@ -11,7 +11,7 @@ Si vous n'avez pas YunoHost, consultez [le guide](https://yunohost.org/#/install
 ## Vue d'ensemble
 LibrePhotos est un sercice inspiré par Google Photos. Il prend en charge la classification des visages, le groupement des photos par date, la localisation géographique, ou objets présents, et la création d'albums.
 
-**Version incluse :** 2021-05-25
+**Version incluse :** 2021-05-31
 
 ## Captures d'écran
 

conf/backend.src

@@ -1,7 +1,7 @@
-SOURCE_URL=https://github.com/LibrePhotos/librephotos/tarball/483a90bc61196364b2d37ea7749e501cb9de50b7
+SOURCE_URL=https://github.com/LibrePhotos/librephotos/tarball/bd4fabf104ea818a4b8fae61e6e41c08ad89ba16
-SOURCE_SUM=1d838a739f16a994b25f4511334f65c5a246b46a746831611fe4296ffdc117fb
+SOURCE_SUM=6cd32a2d75173c851aed76b8f045f95b67fafac4be9ff499c498c6ceba843eb2
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=true
-SOURCE_FILENAME=483a90bc61196364b2d37ea7749e501cb9de50b7
+SOURCE_FILENAME=bd4fabf104ea818a4b8fae61e6e41c08ad89ba16
 SOURCE_EXTRACT=true

conf/dlib.src

@@ -0,0 +1,7 @@
+SOURCE_URL=https://github.com/davisking/dlib/archive/refs/tags/v19.22.tar.gz
+SOURCE_SUM=5f44b67f762691b92f3e41dcf9c95dd0f4525b59cacb478094e511fdacb5c096
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=tar.gz
+SOURCE_IN_SUBDIR=true
+SOURCE_FILENAME=v19.22.tar.gz
+SOURCE_EXTRACT=true

conf/faiss.src

@@ -0,0 +1,7 @@
+SOURCE_URL=https://github.com/facebookresearch/faiss/archive/refs/tags/v1.7.1.tar.gz
+SOURCE_SUM=d676d3107ad41203a49e0afda2630519299dc8666f8d23322cbe1eac0c431871
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=tar.gz
+SOURCE_IN_SUBDIR=true
+SOURCE_FILENAME=v1.7.1.tar.gz
+SOURCE_EXTRACT=true

conf/frontend.src

@@ -1,7 +1,7 @@
-SOURCE_URL=https://github.com/LibrePhotos/librephotos-frontend/tarball/ab1c9e3fe7157580f440338b5b7b1096c055dea8
+SOURCE_URL=https://github.com/LibrePhotos/librephotos-frontend/tarball/86b2bb61b4a3e43e200d3fe22766b92c73c30e74
-SOURCE_SUM=00df92d233492f6a29c5faf900116fb09646dc099edc268d70a019eb4240e411
+SOURCE_SUM=82ddfd1d698d3d8c4645449ac4c0fc442bae74ddb1a77934ef9516672f4731a6
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=true
-SOURCE_FILENAME=ab1c9e3fe7157580f440338b5b7b1096c055dea8
+SOURCE_FILENAME=86b2bb61b4a3e43e200d3fe22766b92c73c30e74
 SOURCE_EXTRACT=true

conf/linux.src

@@ -1,7 +1,7 @@
-SOURCE_URL=https://github.com/LibrePhotos/librephotos-linux/tarball/a8b77d684708a268485a46bb5c06d59dffc764df
+SOURCE_URL=https://github.com/LibrePhotos/librephotos-linux/tarball/3891c65924dc49d87aea0bc9b79448e9d56cd954
-SOURCE_SUM=5128240de9e543c0c360608528656e35fd828633284457519b6de9cb905b56fa
+SOURCE_SUM=1c6b4e9b12f3bea6e298494ad7880483190b4deefb006c9c8d22f840dad2e1ba
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=true
-SOURCE_FILENAME=a8b77d684708a268485a46bb5c06d59dffc764df
+SOURCE_FILENAME=3891c65924dc49d87aea0bc9b79448e9d56cd954
 SOURCE_EXTRACT=true

conf/miniforge3.src

@@ -0,0 +1,7 @@
+SOURCE_URL=https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh
+SOURCE_SUM=68f11be0b8272b9218f62fa3ba1b7c58783427e65db7b7ba77e7cdf91f099540
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=sh
+SOURCE_IN_SUBDIR=false
+SOURCE_FILENAME=Miniforge3-4.10.1-4-Linux-aarch64.sh
+SOURCE_EXTRACT=false

conf/nginx.conf

@@ -15,11 +15,22 @@ location ~ ^/(api|media)/ {
   #unicorn need to listen on [::1]
   proxy_pass http://127.0.0.1:__BACKEND_PORT__;
 }
+
 # Django media
 location /protected_media  {
     internal;
     alias /home/yunohost.app/__NAME__/protected_media/;
 }
+
+location /static/drf-yasg {
+  proxy_pass http://127.0.0.1:__BACKEND_PORT__;
+}
+
+location /data  {
+    internal;
+    alias /home/yunohost.app/__NAME__/data/;
+}
+
 # Original Photos
 location /original {
     internal;

manifest.json

@@ -6,7 +6,7 @@
         "en": "A photo viewer and manager similar to Google Photos",
         "fr": "Un gestionnaire de photos semblable à Google Photos"
     },
-    "version": "0.0.20210525~ynh1",
+    "version": "0.0.20210531~ynh1",
     "url": "https://github.com/LibrePhotos/librephotos",
     "license": "MIT",
     "maintainer": {

scripts/_common.sh

@@ -5,22 +5,25 @@
 #=================================================
 
 # dependencies used by the app
-pkg_dependencies="acl swig libpq-dev postgresql postgresql-contrib postgresql-common ffmpeg libimage-exiftool-perl curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch"
+pkg_dependencies="unzip ca-certificates swig libpq-dev postgresql postgresql-contrib postgresql-common ffmpeg libimage-exiftool-perl curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx gfortran gunicorn libheif-dev libssl-dev rustc liblzma-dev python3 python3-pip python3-venv imagemagick xsel nodejs npm redis-server libmagickwand-dev libldap2-dev libsasl2-dev ufraw-batch"
+
+arch="$(dpkg --print-architecture)"
 
 #=================================================
 # PERSONAL HELPERS
 #=================================================
 
 function unpack_source {
-	mkdir -p $final_path/data_models/{places365,im2txt}
+	ynh_secure_remove "$final_path"
+	mkdir -p "$final_path/data_models/"{places365,im2txt}
 	ynh_setup_source --source_id="places365_model" --dest_dir="$final_path/data_models/places365/model/"
 	ynh_setup_source --source_id="im2txt_model" --dest_dir="$final_path/data_models/im2txt/model/"
 	ynh_setup_source --source_id="im2txt_data" --dest_dir="$final_path/data_models/im2txt/data/"
-	mkdir -p $data_path
+	mkdir -p "$data_path"
 	ln -sf "$final_path/data_models" "$data_path/data_models"
-	mkdir -p $data_path/protected_media/{thumbnails_big,square_thumbnails,square_thumbnails_small,faces}
+	mkdir -p "$data_path/protected_media/"{thumbnails_big,square_thumbnails,square_thumbnails_small,faces}
-	mkdir -p $data_path/data/nextcloud_media
+	mkdir -p "$data_path/data/nextcloud_media"
-	mkdir -p $data_path/matplotlib
+	mkdir -p "$data_path/matplotlib"
 
 	mkdir -p ~/.cache/torch/hub/checkpoints/
 	ynh_setup_source --source_id="resnet152-b121ed2d" --dest_dir="/root/.cache/torch/hub/checkpoints/"
@@ -28,30 +31,72 @@ function unpack_source {
 	ynh_setup_source --source_id="backend" --dest_dir="$final_path/backend/"
 	ynh_setup_source --source_id="frontend" --dest_dir="$final_path/frontend/"
 	ynh_setup_source --source_id="linux" --dest_dir="$final_path/linux/"
+	ynh_setup_source --source_id="dlib" --dest_dir="$final_path/backend/dlib/"
+	if [ "$arch" = "aarch64" ]; then
+		export CONDA_DIR="$final_path/backend/conda"
+		mkdir -p "$CONDA_DIR"
+		ynh_setup_source --source_id="miniforge3" --dest_dir="$CONDA_DIR"
+		ynh_setup_source --source_id="faiss" --dest_dir="$final_path/backend/faiss/"
+	fi
 	mkdir -p "$final_path/bin"
 	mv -f "$final_path/linux/ressources/bin/"* "$final_path/bin"
 	ynh_secure_remove --file="$final_path/linux"
 
-	mkdir -p /var/log/$app
+	mkdir -p "/var/log/$app"
 }
 
 function set_up_virtualenv {
-	backend_path=$final_path/backend
+	backend_path="$final_path/backend"
-	pushd $backend_path || ynh_die
+	pushd "$backend_path"
-		chown -R $app:$app $backend_path
+		chown -R $app:$app "$backend_path"
 		sudo -u $app python3 -m venv $backend_path/venv
-		sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U wheel pip setuptools 2>&1
+		local python_path="$backend_path/venv/bin:$(sudo -u $app bash -c 'echo $PATH')"
-		sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.8.0+cpu torchvision==0.9.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1
+		local cache_dir="$backend_path/.cache/pip"
-		sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib 2>&1
+		sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U wheel pip setuptools 2>&1
-		sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements.txt 2>&1
+		if [ "$arch" = "aarch64" ]; then
-		sudo -u $app $backend_path/venv/bin/pip --cache-dir $backend_path/.cache/pip install -U --requirement $backend_path/requirements-ynh.txt 2>&1
+			sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://torch.maku.ml/whl/stable.html 2>&1
-		sudo -u $app $backend_path/venv/bin/python -m spacy download en_core_web_sm 2>&1
+			sudo -u $app env "CONDA_DIR=$CONDA_DIR" bash "${CONDA_DIR}/Miniforge3-4.10.1-4-Linux-aarch64.sh" -b -p "${CONDA_DIR}"
-		chown -R root:root $backend_path
+			local conda="$CONDA_DIR/condabin/conda"
-	popd || ynh_die
+			python_path="$CONDA_DIR/condabin:$CONDA_DIR/bin:$python_path"
+			sudo -u $app env "PATH=$python_path" conda install -y numpy psycopg2 cython pandas scikit-learn=0.24.1 scikit-image=0.18.1 spacy=2.3.5 gevent=20.12.1 matplotlib=3.3.2
+			pushd "$backend_path/faiss"
+				sudo -u $app cmake -B build .
+				sudo -u $app make -C build -j faiss
+				sudo -u $app make -C build -j swigfaiss
+				cd "build/faiss/python"
+				sudo -u $app env "PATH=$python_path" python setup.py install
+			popd
+			sed -i "/spacy==2.3.2/d" "$backend_path/requirements.txt"
+			sed -i "/sklearn==0.0/d" "$backend_path/requirements.txt"
+			sed -i "/gevent==20.9.0/d" "$backend_path/requirements.txt"
+			sed -i "/scipy==1.5.3/d" "$backend_path/requirements.txt"
+			sed -i "s/pytz==2020.1/pytz>=2021.1/" "$backend_path/requirements.txt"
+			sed -i "s/Pillow==8.1.0/Pillow>=8.1.2/" "$backend_path/requirements.txt"
+			sed -i "/faiss-cpu==1.7.0/d" "$backend_path/requirements.txt"
+		else
+			sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U torch==1.8.0+cpu torchvision==0.9.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 2>&1 || sudo -u $app "$pip" --cache-dir "$cache_dir" install -U torch==1.8.0 torchvision==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html 2>&1
+		fi
+		pushd "$backend_path/dlib"
+			sudo -u $app env "PATH=$python_path" python setup.py install 2>&1
+		popd
+		sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements.txt" 2>&1
+		sudo -u $app env "PATH=$python_path" pip --cache-dir "$cache_dir" install -U --requirement "$backend_path/requirements-ynh.txt" 2>&1
+		sudo -u $app env "PATH=$python_path" python -m spacy download en_core_web_sm 2>&1
+		if [ "$arch" = "aarch64" ]; then
+			unzip "$CONDA_DIR/lib/python3.8/site-packages/"faiss*.egg -d "$CONDA_DIR/lib/python3.8/site-packages/"
+		fi
+		chown -R root:root "$backend_path"
+	popd
+}
+
+function install_dlib {
+	pushd "$backend_path/dlib"
+		sudo -u $app "$backend_path/venv/bin/python" setup.py install 2>&1
+	popd
 }
 
 function set_node_vars {
-	ynh_exec_warn_less ynh_install_nodejs --nodejs_version=10
+	ynh_exec_warn_less ynh_install_nodejs --nodejs_version=13
 	ynh_use_nodejs
 	node_PATH=$nodejs_path:$(sudo -u $app sh -c 'echo $PATH')
 
@@ -60,14 +105,14 @@ function set_node_vars {
 function set_up_frontend {
 	set_node_vars
 	frontend_path=$final_path/frontend
-	pushd $final_path/frontend || ynh_die
+	pushd $final_path/frontend
 		chown -R $app:$app $frontend_path
 		sudo -u $app touch $frontend_path/.yarnrc
 		sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc install 2>&1
 		sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc run build 2>&1
 		sudo -u $app env "PATH=$node_PATH" yarn --cache-folder $frontend_path/yarn-cache --use-yarnrc $frontend_path/.yarnrc add serve 2>&1
 		chown -R root:root $frontend_path
-	popd || ynh_die
+	popd
 }
 
 function add_configuations {
@@ -103,15 +148,15 @@ function add_configuations {
 }
 
 function set_permissions {
-	chown -R root:$app $final_path
+	chown -R root:$app "$final_path"
-	chmod -R g=u,g-w,o-rwx $final_path
+	chmod -R g=u,g-w,o-rwx "$final_path"
-	chown -R $app:$app $data_path
+	chown -R $app:$app "$data_path"
-	chmod -R g=u,g-w,o-rwx $data_path
+	chmod -R g=u,g-w,o-rwx "$data_path"
-	chown -R $app:$app $final_path/data_models
+	chown -R $app:$app "$final_path/data_models"
-	chown -R $app:$app /var/log/$app
+	chown -R $app:$app "/var/log/$app"
-	chmod -R g-w,o-rwx /var/log/$app
+	chmod -R g-w,o-rwx "/var/log/$app"
-	setfacl -n -m user:www-data:rx $data_path
+	setfacl -n -m user:www-data:rx "$data_path"
-	setfacl -n -R -m user:www-data:rx -m default:user:www-data:rx $data_path/protected_media $data_path/data $data_path/data/nextcloud_media
+	setfacl -nR -m u:www-data:rx -m d:u:www-data:rx "$data_path/protected_media" "$data_path/data" "$data_path/data/nextcloud_media"
 }
 
 function set_up_logrotate {

scripts/upgrade

@@ -62,6 +62,17 @@ ynh_systemd_action --service_name=$app-frontend --action="stop" --log_path="syst
 ynh_systemd_action --service_name=$app-image-similarity --action="stop" --log_path="/var/log/$app/image_similarity.log"
 ynh_systemd_action --service_name=$app-worker --action="stop" --log_path="/var/log/$app/$app-worker.log"
 
+#=================================================
+# ENSURE DOWNWARD COMPATIBILITY
+#=================================================
+ynh_script_progression --message="Ensuring downward compatibility..." --time --weight=1
+
+# Remove old node version
+ynh_use_nodejs
+if [ "$nodejs_version" -eq 10 ]; then
+	ynh_remove_nodejs
+fi
+
 #=================================================
 # DOWNLOAD, CHECK AND UNPACK SOURCE
 #=================================================