diff --git a/README.md b/README.md index e1f295b..d242c48 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ If you don't have YunoHost, please see [here](https://yunohost.org/#/install) to ## Overview Mastodon is a free, open-source social network. A decentralized alternative to commercial platforms, it avoids the risks of a single company monopolizing your communication. Pick a server that you trust — whichever you choose, you can interact with everyone else. Anyone can run their own Mastodon instance and participate in the social network seamlessly. -**Shipped version:** 2.9.2 +**Shipped version:** 3.1.2 ## Important points to read before installing @@ -28,18 +28,6 @@ Mastodon is a free, open-source social network. A decentralized alternative to c ### Install -#### Adding "swapfile" If you have less than 2Go of RAM -``` -sudo dd if=/dev/zero of=/swapfile bs=1024 count=1024000 -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -``` -add this line on /etc/fstab -``` -/swapfile none swap sw 0 0 -``` - #### Using __screen__ in case of disconnect ``` $ sudo apt-get install screen @@ -72,7 +60,6 @@ LDAP authentication is activated. * x86-64b - [![Build Status](https://ci-apps.yunohost.org/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-apps.yunohost.org/ci/apps/mastodon/) * ARMv8-A - [![Build Status](https://ci-apps-arm.yunohost.org/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-apps-arm.yunohost.org/ci/apps/mastodon/) -* Jessie x86-64b - [![Build Status](https://ci-stretch.nohost.me/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-stretch.nohost.me/ci/apps/mastodon/) ## Links diff --git a/README_fr.md b/README_fr.md index e286481..e1bd115 100644 --- a/README_fr.md +++ b/README_fr.md @@ -11,7 +11,7 @@ Si vous n'avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour ## Vue d'ensemble Mastodon est un réseau social gratuit et open source. Une alternative décentralisée aux plates-formes commerciales, elle évite les risques d'une seule société qui monopolise votre communication. Choisissez un serveur sur lequel vous faites confiance - selon votre choix, vous pouvez interagir avec tous les autres. N'importe qui peut exécuter sa propre instance de Mastodon et participer au réseau social de façon transparente. -**Version incluse:** 2.9.2 +**Version incluse:** 3.1.2 ## Points importants à lire avant l'installation @@ -28,17 +28,6 @@ Mastodon est un réseau social gratuit et open source. Une alternative décentra ### Installation -#### Ajout d'un "swapfile" si vous avez moins de 2Go de RAM -``` -sudo dd if=/dev/zero of=/swapfile bs=1024 count=1024000 -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -``` -ajouter cette ligne dans /etc/fstab -``` -/swapfile none swap sw 0 0 -``` #### Utilisation de __screen__ en cas de déconnection ``` $ sudo apt-get install screen @@ -72,7 +61,6 @@ L'authentification LDAP est activée * x86-64b - [![Build Status](https://ci-apps.yunohost.org/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-apps.yunohost.org/ci/apps/mastodon/) * ARMv8-A - [![Build Status](https://ci-apps-arm.yunohost.org/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-apps-arm.yunohost.org/ci/apps/mastodon/) -* Jessie x86-64b - [![Build Status](https://ci-stretch.nohost.me/ci/logs/mastodon%20%28Apps%29.svg)](https://ci-stretch.nohost.me/ci/apps/mastodon/) ## Links diff --git a/check_process b/check_process index b4eef2b..66a0890 100644 --- a/check_process +++ b/check_process @@ -12,7 +12,6 @@ setup_private=0 setup_public=1 upgrade=1 - upgrade=1 from_commit=2987ccf8b3aa393dbef79b874dfe5839a0ee10da backup_restore=1 multi_instance=1 # This test is no longer necessary since the version 2.7 (PR: https://github.com/YunoHost/yunohost/pull/304), you can still do it if your app could be installed with this version. diff --git a/conf/.env.production.sample b/conf/.env.production.sample index c55d99b..d2f4b91 100644 --- a/conf/.env.production.sample +++ b/conf/.env.production.sample @@ -69,6 +69,7 @@ SMTP_PORT=25 #SMTP_LOGIN= #SMTP_PASSWORD= SMTP_FROM_ADDRESS=__SMTP_FROM_ADDRESS__ +#SMTP_REPLY_TO= #SMTP_DOMAIN= # defaults to LOCAL_DOMAIN SMTP_DELIVERY_METHOD=sendmail # delivery method can also be smtp SMTP_AUTH_METHOD=none @@ -114,6 +115,20 @@ SMTP_OPENSSL_VERIFY_MODE=none # S3_ENDPOINT= # S3_SIGNATURE_VERSION= +# Google Cloud Storage (optional) +# Use S3 compatible API. Since GCS does not support Multipart Upload, +# increase the value of S3_MULTIPART_THRESHOLD to disable Multipart Upload. +# The attachment host must allow cross origin request - see the description +# above. +# S3_ENABLED=true +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# S3_REGION= +# S3_PROTOCOL=https +# S3_HOSTNAME=storage.googleapis.com +# S3_ENDPOINT=https://storage.googleapis.com +# S3_MULTIPART_THRESHOLD=52428801 # 50.megabytes + # Swift (optional) # The attachment host must allow cross origin request - see the description # above. @@ -208,8 +223,8 @@ LDAP_TLS_NO_VERIFY=true # Optional SAML authentication (cf. omniauth-saml) # SAML_ENABLED=true -# SAML_ACS_URL= -# SAML_ISSUER=http://localhost:3000/auth/auth/saml/callback +# SAML_ACS_URL=http://localhost:3000/auth/auth/saml/callback +# SAML_ISSUER=https://example.com # SAML_IDP_SSO_TARGET_URL=https://idp.testshib.org/idp/profile/SAML2/Redirect/SSO # SAML_IDP_CERT= # SAML_IDP_CERT_FINGERPRINT= @@ -232,3 +247,13 @@ LDAP_TLS_NO_VERIFY=true # http_proxy=http://gateway.local:8118 # Access control for hidden service. # ALLOW_ACCESS_TO_HIDDEN_SERVICE=true + +# Authorized fetch mode (optional) +# Require remote servers to authentify when fetching toots, see +# https://docs.joinmastodon.org/admin/config/#authorized_fetch +# AUTHORIZED_FETCH=true + +# Whitelist mode (optional) +# Only allow federation with whitelisted domains, see +# https://docs.joinmastodon.org/admin/config/#whitelist_mode +# WHITELIST_MODE=true diff --git a/conf/app.src b/conf/app.src index 0973ab9..fc275ee 100644 --- a/conf/app.src +++ b/conf/app.src @@ -1,5 +1,5 @@ -SOURCE_URL=https://github.com/tootsuite/mastodon/archive/v2.9.2.tar.gz -SOURCE_SUM=b46f5f2b2032b531997acf942005ce5f88c9389fc3e5b307a281854179d64878 +SOURCE_URL=https://github.com/tootsuite/mastodon/archive/v3.1.2.tar.gz +SOURCE_SUM=c12fd8c1b426861825d37d2eab1d6356be05b2b321eae0c1b87f80fcc11b0da7 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=true diff --git a/conf/cron b/conf/cron index 6463d70..9a984a3 100644 --- a/conf/cron +++ b/conf/cron @@ -1,2 +1,2 @@ RAILS_ENV=production -@daily cd __FINAL__PATH__/live && /opt/rbenv/versions/2.6.1/bin/bundle exec rake __USER__:media:remove_remote +@daily cd __FINAL__PATH__/live && /opt/rbenv/versions/2.6.5/bin/bundle exec rake __USER__:media:remove_remote diff --git a/conf/mastodon-sidekiq.service b/conf/mastodon-sidekiq.service index cd7db81..d5fc7e1 100644 --- a/conf/mastodon-sidekiq.service +++ b/conf/mastodon-sidekiq.service @@ -9,7 +9,7 @@ Environment="RAILS_ENV=production" Environment="DB_POOL=25" Environment="MALLOC_ARENA_MAX=2" - ExecStart=/opt/rbenv/versions/2.6.1/bin/bundle exec sidekiq -c 25 + ExecStart=/opt/rbenv/versions/2.6.5/bin/bundle exec sidekiq -c 25 TimeoutSec=15 Restart=always StandardError=syslog diff --git a/conf/mastodon-web.service b/conf/mastodon-web.service index d2f5fdd..bb65afe 100644 --- a/conf/mastodon-web.service +++ b/conf/mastodon-web.service @@ -8,7 +8,7 @@ WorkingDirectory=__FINALPATH__/live Environment="RAILS_ENV=production" Environment="PORT=__PORT_WEB__" - ExecStart=/opt/rbenv/versions/2.6.1/bin/bundle exec puma -C config/puma.rb + ExecStart=/opt/rbenv/versions/2.6.5/bin/bundle exec puma -C config/puma.rb ExecReload=/bin/kill -SIGUSR1 $MAINPID TimeoutSec=15 Restart=always diff --git a/conf/nginx.conf b/conf/nginx.conf index 190c650..7fa7b87 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -6,70 +6,70 @@ root __FINALPATH__/live/public; location / { - if ($scheme = http) { - rewrite ^ https://$server_name$request_uri? permanent; - } + # Force usage of https + if ($scheme = http) { + rewrite ^ https://$server_name$request_uri? permanent; + } - proxy_set_header Accept-Encoding ""; - try_files $uri @proxy; + proxy_set_header Accept-Encoding ""; + try_files $uri @proxy; - # Include SSOWAT user panel. - include conf.d/yunohost_panel.conf.inc; + # Include SSOWAT user panel. + include conf.d/yunohost_panel.conf.inc; } location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) { - add_header Cache-Control "public, max-age=31536000, immutable"; - add_header Strict-Transport-Security "max-age=31536000"; - try_files $uri @proxy; + add_header Cache-Control "public, max-age=31536000, immutable"; + add_header Strict-Transport-Security "max-age=31536000"; + try_files $uri @proxy; } location /sw.js { - add_header Cache-Control "public, max-age=0"; - add_header Strict-Transport-Security "max-age=31536000"; - try_files $uri @proxy; + add_header Cache-Control "public, max-age=0"; + add_header Strict-Transport-Security "max-age=31536000"; + try_files $uri @proxy; } location @proxy { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Proxy ""; - proxy_pass_header Server; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Proxy ""; + proxy_pass_header Server; - proxy_pass http://127.0.0.1:3000; - proxy_buffering on; - proxy_redirect off; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; + proxy_pass http://127.0.0.1:3000; + proxy_buffering on; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; - #proxy_cache CACHE; - proxy_cache_valid 200 7d; - proxy_cache_valid 410 24h; - proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; - add_header X-Cached $upstream_cache_status; - add_header Strict-Transport-Security "max-age=31536000"; + #proxy_cache CACHE; + proxy_cache_valid 200 7d; + proxy_cache_valid 410 24h; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + add_header X-Cached $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000"; - tcp_nodelay on; + tcp_nodelay on; } location /api/v1/streaming { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Proxy ""; - - proxy_pass http://127.0.0.1:4000; - proxy_buffering off; - proxy_redirect off; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - tcp_nodelay on; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Proxy ""; + + proxy_pass http://127.0.0.1:4000; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + tcp_nodelay on; } - -error_page 500 501 502 503 504 /500.html; \ No newline at end of file +error_page 500 501 502 503 504 /500.html; diff --git a/manifest.json b/manifest.json index d21d9c5..8c7c3d4 100644 --- a/manifest.json +++ b/manifest.json @@ -6,7 +6,7 @@ "en": "Mastodon is a free, open-source social network.", "fr": "Mastodon est un réseau social gratuit et open source." }, - "version": "2.9.2~ynh1", + "version": "3.1.2~ynh2", "url": "https://github.com/tootsuite/mastodon", "license": "AGPL-3.0-or-later", "maintainer": [ diff --git a/scripts/_common.sh b/scripts/_common.sh index 904553d..015c1e9 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -5,8 +5,7 @@ #================================================= # dependencies used by the app -#pkg_dependencies="imagemagick ffmpeg libpq-dev libxml2-dev libxslt1-dev file git-core g++ libprotobuf-dev protobuf-compiler pkg-config nodejs gcc autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm5 libgdbm-dev nginx redis-server redis-tools postgresql postgresql-contrib certbot python-certbot-nginx yarn libidn11-dev libicu-dev libjemalloc-dev" - pkg_dependencies="imagemagick ffmpeg libpq-dev libxml2-dev libxslt1-dev file git-core g++ libprotobuf-dev protobuf-compiler pkg-config gcc autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm3 libgdbm-dev redis-server redis-tools postgresql postgresql-contrib libidn11-dev libicu-dev libjemalloc-dev curl apt-transport-https" +pkg_dependencies="imagemagick ffmpeg libpq-dev libxml2-dev libxslt1-dev file git-core g++ libprotobuf-dev protobuf-compiler pkg-config gcc autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm3|libgdbm6 libgdbm-dev redis-server redis-tools postgresql postgresql-contrib libidn11-dev libicu-dev libjemalloc-dev curl apt-transport-https" #================================================= # PERSONAL HELPERS diff --git a/scripts/backup b/scripts/backup index 62ac387..61ad5f4 100644 --- a/scripts/backup +++ b/scripts/backup @@ -13,6 +13,7 @@ source /usr/share/yunohost/helpers #================================================= # MANAGE SCRIPT FAILURE #================================================= +ynh_script_progression --message="Managing script failure..." --weight=1 ynh_clean_setup () { ynh_clean_check_starting @@ -77,6 +78,7 @@ ynh_backup --src_path="/etc/systemd/system/$app-streaming.service" #================================================= # BACKUP A CRON FILE #================================================= +ynh_script_progression --message="Backing up a cron file..." --weight=1 ynh_backup --src_path="/etc/cron.d/$app" diff --git a/scripts/install b/scripts/install index 55b1480..cb059ba 100644 --- a/scripts/install +++ b/scripts/install @@ -7,13 +7,17 @@ #================================================= source _common.sh -source ynh_install_ruby +source ynh_install_ruby__2 source ynh_add_extra_apt_repos__3 +source ynh_send_readme_to_admin__2 +source ynh_add_swap +source ynh_check_ram source /usr/share/yunohost/helpers #================================================= # MANAGE SCRIPT FAILURE #================================================= +ynh_script_progression --message="Managing script failure..." --weight=1 ynh_clean_setup () { ynh_clean_check_starting @@ -24,6 +28,7 @@ ynh_abort_if_errors #================================================= # RETRIEVE ARGUMENTS FROM THE MANIFEST #================================================= +ynh_script_progression --message="Retrieving arguments from the manifest..." --weight=1 domain=$YNH_APP_ARG_DOMAIN path_url="/" @@ -43,16 +48,6 @@ ynh_script_progression --message="Validating installation parameters..." --weigh final_path=/var/www/$app test ! -e "$final_path" || ynh_die --message="This path already contains a folder" -if [ ${PACKAGE_CHECK_EXEC:-0} -eq 1 ]; then - # TODO : to be factorized into a helper someday ? ;) - MEM=$(free | grep "^Mem" | awk '{print $2}') - SWAP=$(free | grep "^Swap" | awk '{print $2}') - TOTAL_MEM_AND_SWAP=$(( ( $MEM+$SWAP ) / 1024 )) # In MB - - [[ $TOTAL_MEM_AND_SWAP -gt 2500 ]] || ynh_die "You need at least 2500 Mo of RAM+Swap to install Mastodon. Please consult the README to learn how to add swap." - -fi - # Register (book) web path ynh_webpath_register --app=$app --domain=$domain --path_url=$path_url @@ -74,7 +69,7 @@ ynh_app_setting_set --app=$app --key=language --value=$language #================================================= ynh_script_progression --message="Configuring firewall..." --weight=1 -# Find a free port +# Find an available port port_web=$(ynh_find_port --port=3000) port_stream=$(ynh_find_port --port=4000) # Open this port @@ -86,7 +81,7 @@ ynh_app_setting_set --app=$app --key=port_stream --value=$port_stream #================================================= ynh_script_progression --message="Installing dependencies..." --weight=86 -ynh_install_nodejs --nodejs_version="8" +ynh_install_nodejs --nodejs_version="10" ynh_install_app_dependencies $pkg_dependencies ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ stable main" --package="yarn" --key="https://dl.yarnpkg.com/debian/pubkey.gpg" @@ -98,11 +93,12 @@ ynh_script_progression --message="Creating a PostgreSQL database..." --weight=5 # Create postgresql database db_name="${app}_production" db_user=$app -db_pwd=$(ynh_string_random 30) +db_pwd=$(ynh_string_random --length=30) ynh_app_setting_set --app=$app --key=db_name --value=$db_name ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd ynh_psql_test_if_first_run ynh_psql_setup_db --db_user=$db_user --db_name=$db_name --db_pwd=$db_pwd +ynh_psql_execute_as_root --sql="ALTER USER $db_user CREATEDB;" #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE @@ -114,6 +110,9 @@ ynh_app_setting_set --app=$app --key=final_path --value=$final_path mkdir $final_path ynh_setup_source --dest_dir="$final_path/live" +# Temporary workaround for https://github.com/tootsuite/mastodon/issues/13292 +ynh_replace_string --match_string="sidekiq-unique-jobs (6.0.18)" --replace_string="sidekiq-unique-jobs (6.0.20)" --target_file="$final_path/live/Gemfile.lock" + #================================================= # NGINX CONFIGURATION #================================================= @@ -132,14 +131,31 @@ ynh_system_user_create --username=$app --home_dir=$final_path #================================================= # SPECIFIC SETUP +#================================================= +# ADD SWAP IF NEEDED +#================================================= +ynh_script_progression --message="Adding swap is needed..." --weight=4 + +total_memory=$(ynh_check_ram) +total_swap=$(ynh_check_ram --only_swap) +swap_needed=0 + +if [ $total_memory -lt 2560 ]; then + # Need a minimum of 8Go of memory + swap_needed=$((2560 - $total_memory)) +fi + +ynh_script_progression --message="Adding $swap_needed Mo to swap..." --weight=1 +ynh_add_swap --size=$swap_needed + #================================================= # INSTALLING RUBY AND BUNDLER #================================================= ynh_script_progression --message="Installing Ruby..." --weight=424 -ynh_install_ruby --ruby_version=2.6.1 -/opt/rbenv/versions/2.6.1/bin/gem update --system -#/opt/rbenv/versions/2.6.1/bin/gem install bundler --no-document +ynh_install_ruby --ruby_version=2.6.5 +/opt/rbenv/versions/2.6.5/bin/gem update --system +/opt/rbenv/versions/2.6.5/bin/gem install bundler:1.17.3 --no-document #================================================= # MODIFY A CONFIG FILE @@ -177,17 +193,20 @@ chown -R "$app": "$final_path" pushd "$final_path/live" ynh_use_nodejs - sudo -u "$app" env PATH=$PATH /opt/rbenv/versions/2.6.1/bin/bundle install -j$(getconf _NPROCESSORS_ONLN) --deployment --without development test + sudo -u "$app" env PATH=$PATH /opt/rbenv/versions/2.6.5/bin/bundle install -j$(getconf _NPROCESSORS_ONLN) --deployment --without development test sudo -u "$app" env PATH=$PATH yarn install --pure-lockfile sudo -u "$app" echo "SAFETY_ASSURED=1">> .env.production - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rails db:migrate --quiet - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rails assets:precompile --quiet - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rake mastodon:webpush:generate_vapid_key > key.txt - sudo -u "$app" env PATH=$PATH RAILS_ENV=production bin/tootctl accounts create "$admin" --email="$admin_mail" --confirmed --role=admin > /dev/null 2>&1 + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rails db:setup --quiet + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rails assets:precompile --quiet + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rake mastodon:webpush:generate_vapid_key > key.txt + sudo -u "$app" env PATH=$PATH RAILS_ENV=production bin/tootctl accounts create "$admin" --email="$admin_mail" --confirmed --role=admin > acc.txt popd -vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K\w+" "$final_path/live/key.txt") -vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K\w+" "$final_path/live/key.txt") +admin_pass=$( tail -1 $final_path/live/acc.txt | head -1 | cut -c 15- ) +ynh_secure_remove --file="$final_path/live/acc.txt" + +vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "$final_path/live/key.txt") +vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "$final_path/live/key.txt") ynh_replace_string --match_string="__VAPID_PRIVATE_KEY__" --replace_string="$vapid_private_key" --target_file="${final_path}/live/.env.production" ynh_replace_string --match_string="__VAPID_PUBLIC_KEY__" --replace_string="$vapid_public_key" --target_file="${final_path}/live/.env.production" @@ -238,9 +257,9 @@ ynh_script_progression --message="Securing files and directories..." --weight=9 chown -R "$app": "$final_path" #================================================= -# ADVERTISE SERVICE IN ADMIN PANEL +# INTEGRATE SERVICE IN YUNOHOST #================================================= -ynh_script_progression --message="Advertising service in admin panel..." --weight=3 +ynh_script_progression --message="Integrating service in YunoHost..." --weight=3 yunohost service add "$app-web" yunohost service add "$app-sidekiq" diff --git a/scripts/remove b/scripts/remove index 19cbf29..f2b2e16 100644 --- a/scripts/remove +++ b/scripts/remove @@ -7,8 +7,9 @@ #================================================= source _common.sh -source ynh_install_ruby +source ynh_install_ruby__2 source ynh_add_extra_apt_repos__3 +source ynh_add_swap source /usr/share/yunohost/helpers #================================================= @@ -26,23 +27,24 @@ final_path=$(ynh_app_setting_get --app=$app --key=final_path) #================================================= # STANDARD REMOVE #================================================= -# REMOVE SERVICE FROM ADMIN PANEL +# REMOVE SERVICE INTEGRATION IN YUNOHOST #================================================= +ynh_script_progression --message="Removing service integration in YunoHost..." --weight=2 -# Remove a service from the admin panel, added by `yunohost service add` -if yunohost service status "$app-web" >/dev/null 2>&1 +# Remove the service from the list of services known by Yunohost (added from `yunohost service add`) +if ynh_exec_warn_less yunohost service status "$app-web" >/dev/null then ynh_script_progression --message="Removing $app-web service..." --weight=2 yunohost service remove "$app-web" fi -if yunohost service status "$app-sidekiq" >/dev/null 2>&1 +if ynh_exec_warn_less yunohost service status "$app-sidekiq" >/dev/null then ynh_script_progression --message="Removing $app-sidekiq service..." --weight=2 yunohost service remove "$app-sidekiq" fi -if yunohost service status "$app-streaming" >/dev/null 2>&1 +if ynh_exec_warn_less yunohost service status "$app-streaming" >/dev/null then ynh_script_progression --message="Removing $app-streaming service..." --weight=2 yunohost service remove "$app-streaming" @@ -98,10 +100,14 @@ ynh_remove_nginx_config #================================================= # REMOVE THE CRON FILE #================================================= +ynh_script_progression --message="Removing the cron file..." --weight=2 # Remove a cron file ynh_secure_remove --file="/etc/cron.d/$app" +# Remove swap +ynh_del_swap + #================================================= # GENERIC FINALIZATION #================================================= diff --git a/scripts/restore b/scripts/restore index 59e71fc..c6e4bc8 100644 --- a/scripts/restore +++ b/scripts/restore @@ -8,13 +8,16 @@ #Keep this path for calling _common.sh inside the execution's context of backup and restore scripts source ../settings/scripts/_common.sh -source ../settings/scripts/ynh_install_ruby +source ../settings/scripts/ynh_install_ruby__2 source ../settings/scripts/ynh_add_extra_apt_repos__3 +source ../settings/scripts/ynh_add_swap +source ../settings/scripts/ynh_check_ram source /usr/share/yunohost/helpers #================================================= # MANAGE SCRIPT FAILURE #================================================= +ynh_script_progression --message="Managing script failure..." --weight=2 ynh_clean_setup () { ynh_clean_check_starting @@ -80,12 +83,29 @@ chown -R $app: $final_path #================================================= # SPECIFIC RESTORATION +#================================================= +# ADD SWAP IF NEEDED +#================================================= +ynh_script_progression --message="Adding swap if needed..." --weight=4 + +total_memory=$(ynh_check_ram) +total_swap=$(ynh_check_ram --only_swap) +swap_needed=0 + +if [ $total_memory -lt 2560 ]; then + # Need a minimum of 8Go of memory + swap_needed=$((2560 - $total_memory)) +fi + +ynh_script_progression --message="Adding $swap_needed Mo to swap..." --weight=1 +ynh_add_swap --size=$swap_needed + #================================================= # REINSTALL DEPENDENCIES #================================================= ynh_script_progression --message="Reinstalling dependencies..." --weight=63 -ynh_install_nodejs --nodejs_version="8" +ynh_install_nodejs --nodejs_version="10" ynh_install_app_dependencies $pkg_dependencies ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ stable main" --package="yarn" --key="https://dl.yarnpkg.com/debian/pubkey.gpg" @@ -94,8 +114,9 @@ ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ st #================================================= ynh_script_progression --message="Installing Ruby..." --weight=393 -ynh_install_ruby --ruby_version=2.6.1 -/opt/rbenv/versions/2.6.1/bin/gem update --system +ynh_install_ruby --ruby_version=2.6.5 +/opt/rbenv/versions/2.6.5/bin/gem update --system +/opt/rbenv/versions/2.6.5/bin/gem install bundler:1.17.3 --no-document #================================================= # RESTORE THE POSTGRESQL DATABASE @@ -104,6 +125,7 @@ ynh_script_progression --message="Restoring the PostgreSQL database..." --weight ynh_psql_test_if_first_run ynh_psql_setup_db --db_user=$db_user --db_name=$db_name --db_pwd=$db_pwd +ynh_psql_execute_as_root --sql="ALTER USER $db_user CREATEDB;" ynh_psql_execute_file_as_root --file="./db.sql" --database="$db_name" #================================================= @@ -117,9 +139,9 @@ ynh_restore_file --origin_path="/etc/systemd/system/$app-streaming.service" systemctl enable "$app-web" "$app-sidekiq" "$app-streaming" #================================================= -# ADVERTISE SERVICE IN ADMIN PANEL +# INTEGRATE SERVICE IN YUNOHOST #================================================= -ynh_script_progression --message="Advertising service in admin panel..." --weight=3 +ynh_script_progression --message="Integrating service in YunoHost..." --weight=3 yunohost service add $app-web yunohost service add $app-sidekiq diff --git a/scripts/upgrade b/scripts/upgrade index 46e7229..0bc44b3 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -7,8 +7,10 @@ #================================================= source _common.sh -source ynh_install_ruby +source ynh_install_ruby__2 source ynh_add_extra_apt_repos__3 +source ynh_add_swap +source ynh_check_ram source /usr/share/yunohost/helpers #================================================= @@ -40,6 +42,7 @@ vapid_public_key=$(ynh_app_setting_get --app=$app --key=vapid_public_key) #================================================= # CHECK VERSION #================================================= +ynh_script_progression --message="Checking version..." --weight=1 upgrade_type=$(ynh_check_app_version_changed) @@ -116,8 +119,8 @@ fi # If vapid_private_key doesn't exist, retrieve it or create it if [[ -z "$vapid_private_key" ]]; then - vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K\w+" ${final_path}/live/.env.production) - vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K\w+" ${final_path}/live/.env.production) + vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" ${final_path}/live/.env.production) + vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" ${final_path}/live/.env.production) ynh_app_setting_set "$app" vapid_private_key "$vapid_private_key" ynh_app_setting_set "$app" vapid_public_key "$vapid_public_key" fi @@ -145,6 +148,11 @@ ynh_abort_if_errors #================================================= # Normalize the URL path syntax +# N.B. : this is for app installations before YunoHost 2.7 +# where this value might be something like /foo/ or foo/ +# instead of /foo .... +# If nobody installed your app before 2.7, then you may +# safely remove this line path_url=$(ynh_normalize_url_path --path_url=$path_url) #================================================= @@ -167,13 +175,24 @@ then ynh_script_progression --message="Upgrading source files..." --weight=14 # Download Mastodon - mv "$final_path/live" "$final_path/live_back" + tmpdir="$(mktemp -d)" + + mkdir $tmpdir/system + if [ -d "$final_path/live/public/system" ]; then + rsync -a "$final_path/live/public/system" "$tmpdir/." + fi + rsync -a "$final_path/live/.env.production" "$tmpdir/." + ynh_secure_remove --file="$final_path/live/" ynh_setup_source --dest_dir="$final_path/live" - if [ -d "$final_path/live_back/public/system" ]; then - rsync -a "$final_path/live_back/public/system" "$final_path/live/public/." + + # Temporary workaround for https://github.com/tootsuite/mastodon/issues/13292 + ynh_replace_string --match_string="sidekiq-unique-jobs (6.0.18)" --replace_string="sidekiq-unique-jobs (6.0.20)" --target_file="$final_path/live/Gemfile.lock" + + if [ -d "$tmpdir/system" ]; then + rsync -a "$tmpdir/system" "$final_path/live/public/." fi - rsync -a "$final_path/live_back/.env.production" "$final_path/live/." - ynh_secure_remove --file="$final_path/live_back/" + rsync -a "$tmpdir/.env.production" "$final_path/live/." + ynh_secure_remove --file="$tmpdir/" # Clean files which are not needed anymore ynh_secure_remove --file="$final_path/live/config/initializers/timeout.rb" @@ -191,7 +210,8 @@ ynh_add_nginx_config 'port_web port_stream' #================================================= ynh_script_progression --message="Upgrading dependencies..." --weight=24 -ynh_install_nodejs --nodejs_version="8" +ynh_remove_nodejs +ynh_install_nodejs --nodejs_version="10" ynh_install_app_dependencies $pkg_dependencies ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ stable main" --package="yarn" --key="https://dl.yarnpkg.com/debian/pubkey.gpg" @@ -205,14 +225,31 @@ ynh_system_user_create --username=$app --home_dir=$final_path #================================================= # SPECIFIC UPGRADE +#================================================= +# ADD SWAP IF NEEDED +#================================================= +ynh_script_progression --message="Adding swap if needed..." --weight=7 + +total_memory=$(ynh_check_ram) +total_swap=$(ynh_check_ram --only_swap) +swap_needed=0 + +if [ $total_memory -lt 2560 ]; then + # Need a minimum of 8Go of memory + swap_needed=$((2560 - $total_memory)) +fi + +ynh_script_progression --message="Adding $swap_needed Mo to swap..." --weight=1 +ynh_add_swap --size=$swap_needed + #================================================= # INSTALLING RUBY AND BUNDLER #================================================= ynh_script_progression --message="Installing Ruby..." --weight=424 -ynh_install_ruby --ruby_version=2.6.1 -/opt/rbenv/versions/2.6.1/bin/gem update --system -#/opt/rbenv/versions/2.6.1/bin/gem install bundler +ynh_install_ruby --ruby_version=2.6.5 +/opt/rbenv/versions/2.6.5/bin/gem update --system +/opt/rbenv/versions/2.6.5/bin/gem install bundler:1.17.3 --no-document #================================================= # MODIFY A CONFIG FILE @@ -248,23 +285,19 @@ chown -R "$app": "$final_path" pushd "$final_path/live" ynh_use_nodejs - if [ "$(lsb_release --codename --short)" == "jessie" ]; then - sudo -u "$app" env PATH=$PATH /opt/rbenv/versions/2.6.1/bin/bundle install --deployment --without development test - else - sudo -u "$app" env PATH=$PATH /opt/rbenv/versions/2.6.1/bin/bundle install --deployment --force --without development test - fi - sudo -u "$app" env PATH=$PATH yarn install --pure-lockfile - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rails assets:clean - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rails assets:precompile - sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.1/bin/bundle exec rails db:migrate - sudo -u "$app" env PATH=$PATH RAILS_ENV=production bin/tootctl cache clear + sudo -u "$app" env PATH=$PATH /opt/rbenv/versions/2.6.5/bin/bundle install -j$(getconf _NPROCESSORS_ONLN) --deployment --without development test + sudo -u "$app" env PATH=$PATH yarn install --pure-lockfile + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rails assets:clean + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rails assets:precompile + sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.5/bin/bundle exec rails db:migrate + sudo -u "$app" env PATH=$PATH RAILS_ENV=production bin/tootctl cache clear popd # If vapid_private_key doesn't exist, retrieve it or create it #if [[ -z "$vapid_private_key" ]]; then # sudo -u "$app" env PATH=$PATH RAILS_ENV=production /opt/rbenv/versions/2.6.0/bin/bundle exec rake mastodon:webpush:generate_vapid_key > key.txt -# vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K\w+" "$final_path/live/key.txt") -# vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K\w+" "$final_path/live/key.txt") +# vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "$final_path/live/key.txt") +# vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "$final_path/live/key.txt") # ynh_app_setting_set "$app" vapid_private_key "$vapid_private_key" # ynh_app_setting_set "$app" vapid_public_key "$vapid_public_key" # ynh_secure_remove "$final_path/live/key.txt" diff --git a/scripts/ynh_add_swap b/scripts/ynh_add_swap new file mode 100644 index 0000000..d7ec44b --- /dev/null +++ b/scripts/ynh_add_swap @@ -0,0 +1,93 @@ +#!/bin/bash + +# Add swap +# +# usage: ynh_add_swap --size=SWAP in Mb +# | arg: -s, --size= - Amount of SWAP to add in Mb. +ynh_add_swap () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [s]=size= ) + local size + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + local swap_max_size=$(( $size * 1024 )) + + local free_space=$(df --output=avail / | sed 1d) + # Because we don't want to fill the disk with a swap file, divide by 2 the available space. + local usable_space=$(( $free_space / 2 )) + + SD_CARD_CAN_SWAP=${SD_CARD_CAN_SWAP:-0} + + # Swap on SD card only if it's is specified + if ynh_is_main_device_a_sd_card && [ "$SD_CARD_CAN_SWAP" == "0" ] + then + ynh_print_warn --message="The main mountpoint of your system '/' is on an SD card, swap will not be added to prevent some damage of this one, but that can cause troubles for the app $app. If you still want activate the swap, you can relaunch the command preceded by 'SD_CARD_CAN_SWAP=1'" + return + fi + + # Compare the available space with the size of the swap. + # And set a acceptable size from the request + if [ $usable_space -ge $swap_max_size ] + then + local swap_size=$swap_max_size + elif [ $usable_space -ge $(( $swap_max_size / 2 )) ] + then + local swap_size=$(( $swap_max_size / 2 )) + elif [ $usable_space -ge $(( $swap_max_size / 3 )) ] + then + local swap_size=$(( $swap_max_size / 3 )) + elif [ $usable_space -ge $(( $swap_max_size / 4 )) ] + then + local swap_size=$(( $swap_max_size / 4 )) + else + echo "Not enough space left for a swap file" >&2 + local swap_size=0 + fi + + # If there's enough space for a swap, and no existing swap here + if [ $swap_size -ne 0 ] && [ ! -e /swap_$app ] + then + # Preallocate space for the swap file, fallocate may sometime not be used, use dd instead in this case + if ! fallocate -l ${swap_size}K /swap_$app + then + dd if=/dev/zero of=/swap_$app bs=1024 count=${swap_size} + fi + chmod 0600 /swap_$app + # Create the swap + mkswap /swap_$app + # And activate it + swapon /swap_$app + # Then add an entry in fstab to load this swap at each boot. + echo -e "/swap_$app swap swap defaults 0 0 #Swap added by $app" >> /etc/fstab + fi +} + +ynh_del_swap () { + # If there a swap at this place + if [ -e /swap_$app ] + then + # Clean the fstab + sed -i "/#Swap added by $app/d" /etc/fstab + # Desactive the swap file + swapoff /swap_$app + # And remove it + rm /swap_$app + fi +} + +# Check if the device of the main mountpoint "/" is an SD card +# +# [internal] +# +# return 0 if it's an SD card, else 1 +ynh_is_main_device_a_sd_card () { + local main_device=$(lsblk --output PKNAME --noheadings $(findmnt / --nofsroot --uniq --output source --noheadings --first-only)) + + if echo $main_device | grep --quiet "mmc" && [ $(tail -n1 /sys/block/$main_device/queue/rotational) == "0" ] + then + return 0 + else + return 1 + fi +} diff --git a/scripts/ynh_check_ram b/scripts/ynh_check_ram new file mode 100644 index 0000000..11012a3 --- /dev/null +++ b/scripts/ynh_check_ram @@ -0,0 +1,72 @@ +#!/bin/bash + +# Check the amount of available RAM +# +# usage: ynh_check_ram [--required=RAM required in Mb] [--no_swap|--only_swap] [--free_ram] +# | arg: -r, --required= - Amount of RAM required in Mb. The helper will return 0 is there's enough RAM, or 1 otherwise. +# If --required isn't set, the helper will print the amount of RAM, in Mb. +# | arg: -s, --no_swap - Ignore swap +# | arg: -o, --only_swap - Ignore real RAM, consider only swap. +# | arg: -f, --free_ram - Count only free RAM, not the total amount of RAM available. +ynh_check_ram () { + # Declare an array to define the options of this helper. + declare -Ar args_array=( [r]=required= [s]=no_swap [o]=only_swap [f]=free_ram ) + local required + local no_swap + local only_swap + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + required=${required:-} + no_swap=${no_swap:-0} + only_swap=${only_swap:-0} + + local total_ram=$(vmstat --stats --unit M | grep "total memory" | awk '{print $1}') + local total_swap=$(vmstat --stats --unit M | grep "total swap" | awk '{print $1}') + local total_ram_swap=$(( total_ram + total_swap )) + + local free_ram=$(vmstat --stats --unit M | grep "free memory" | awk '{print $1}') + local free_swap=$(vmstat --stats --unit M | grep "free swap" | awk '{print $1}') + local free_ram_swap=$(( free_ram + free_swap )) + + # Use the total amount of ram + local ram=$total_ram_swap + if [ $free_ram -eq 1 ] + then + # Use the total amount of free ram + ram=$free_ram_swap + if [ $no_swap -eq 1 ] + then + # Use only the amount of free ram + ram=$free_ram + elif [ $only_swap -eq 1 ] + then + # Use only the amount of free swap + ram=$free_swap + fi + else + if [ $no_swap -eq 1 ] + then + # Use only the amount of free ram + ram=$total_ram + elif [ $only_swap -eq 1 ] + then + # Use only the amount of free swap + ram=$total_swap + fi + fi + + if [ -n "$required" ] + then + # Return 1 if the amount of ram isn't enough. + if [ $ram -lt $required ] + then + return 1 + else + return 0 + fi + + # If no RAM is required, return the amount of available ram. + else + echo $ram + fi +} diff --git a/scripts/ynh_install_ruby b/scripts/ynh_install_ruby__2 similarity index 92% rename from scripts/ynh_install_ruby rename to scripts/ynh_install_ruby__2 index 9e53c06..f064c08 100644 --- a/scripts/ynh_install_ruby +++ b/scripts/ynh_install_ruby__2 @@ -15,15 +15,15 @@ ynh_install_rbenv () { echo "Installation of rbenv - ruby version management" >&2 # Build an app.src for rbenv mkdir -p "../conf" - echo "SOURCE_URL=https://github.com/rbenv/rbenv/archive/v1.1.1.tar.gz -SOURCE_SUM=41f1a60714c55eceb21d692a469aee1ec4f46bba351d0dfcb0c660ff9cf1a1c9" > "../conf/rbenv.src" + echo "SOURCE_URL=https://github.com/rbenv/rbenv/archive/v1.1.2.tar.gz +SOURCE_SUM=80ad89ffe04c0b481503bd375f05c212bbc7d44ef5f5e649e0acdf25eba86736" > "../conf/rbenv.src" # Download and extract rbenv ynh_setup_source "$rbenv_install_dir" rbenv # Build an app.src for ruby-build mkdir -p "../conf" - echo "SOURCE_URL=https://github.com/rbenv/ruby-build/archive/v20190314.tar.gz -SOURCE_SUM=2cc0f9fdb232042e71edad93a5c3ae108bcd090ea0b6db4e5bb6325547e07968" > "../conf/ruby-build.src" + echo "SOURCE_URL=https://github.com/rbenv/ruby-build/archive/v20191004.tar.gz +SOURCE_SUM=6f053957acb0af6d621ebf2b9dacc9c265844b2dc6842a021eb10f0a70094fe8" > "../conf/ruby-build.src" # Download and extract ruby-build ynh_setup_source "$rbenv_install_dir/plugins/ruby-build" ruby-build @@ -77,6 +77,9 @@ ynh_install_ruby () { # If rbenv is not previously setup, install it if ! type rbenv > /dev/null 2>&1 + then + ynh_install_rbenv + elif dpkg --compare-versions "$(/opt/rbenv/bin/rbenv --version | cut -d" " -f2)" lt "1.1.2" then ynh_install_rbenv fi