From 0963cb26688f78d9948b7b4195e9a4f716c3afc7 Mon Sep 17 00:00:00 2001 From: orhtej2 <2871798+orhtej2@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:37:28 +0200 Subject: [PATCH] Initial attempt --- conf/sync.ini | 7 +- manifest.toml | 64 ++++------- scripts/_common.sh | 2 + scripts/_common_sh | 17 --- scripts/install | 60 ++++++---- scripts/ynh_install_go | 247 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 313 insertions(+), 84 deletions(-) delete mode 100644 scripts/_common_sh create mode 100644 scripts/ynh_install_go diff --git a/conf/sync.ini b/conf/sync.ini index 65acca3..8e37223 100644 --- a/conf/sync.ini +++ b/conf/sync.ini @@ -1 +1,6 @@ -HOST%3D0.0.0.0%0D%0APORT%3D__PORT__%0D%0AMASTER_SECRET%3D__SHARED_SECRET__%0D%0A%0D%0A%5Bsyncstorage%5D%0D%0ADATABASE_URL%3Dmysql%3A%2F%2F__DB_USER__%3A__DB_PWD__%40127.0.0.1%2F__DB_NAME__ \ No newline at end of file +HOST=0.0.0.0 +PORT=__PORT__ +MASTER_SECRET=__SECRET__ + +[syncstorage] +DATABASE_URL=mysql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__ \ No newline at end of file diff --git a/manifest.toml b/manifest.toml index 25fda46..fd8ea74 100644 --- a/manifest.toml +++ b/manifest.toml @@ -3,36 +3,27 @@ packaging_format = 2 id = "syncstorage-rs" name = "Firefox Sync Storage-rs" -description.en = "New Firefox Sync Storage implemented in Rust. - -Allows syncing tabs, passwords, history etc. between Firefox instances." -description.fr = "New Firefox Sync Storage implemented in Rust." +[description] + en = "New Firefox Sync Storage implemented in Rust. Allows syncing tabs, passwords, history etc. between Firefox instances." + fr = "New Firefox Sync Storage implemented in Rust." version = "0.13.6~ynh1" maintainers = [""] [upstream] -# NB: Only the "license" key is mandatory. Remove entries for which there's no relevant data license = "MPL 2.0" website = "https://www.mozilla.org/en-GB/firefox/features/sync/" -demo = "" admindoc = "https://github.com/mozilla-services/syncstorage-rs/blob/master/docs/config.md" -userdoc = "" code = "https://github.com/mozilla-services/syncstorage-rs" -# FIXME: optional but recommended if relevant, this is meant to contain the Common Platform Enumeration, which is sort of a standard id for applications defined by the NIST. In particular, YunoHost may use this is in the future to easily track CVE (=security reports) related to apps. The CPE may be obtained by searching here: https://nvd.nist.gov/products/cpe/search. For example, for Nextcloud, the CPE is 'cpe:2.3:a:nextcloud:nextcloud' (no need to include the version number) -cpe = "???" -# FIXME: optional but recommended (or remove if irrelevant / not applicable). This is meant to be an URL where people can financially support this app, especially when its development is based on volunteers and/or financed by its community. YunoHost may later advertise it in the webadmin. -fund = "???" - +fund = "https://donate.mozilla.org/en-GB/" [integration] yunohost = ">= 11.1.21" # List of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64), for example: ["amd64", "i386'] architectures = "['all']" # TODO : handle the "all" option (no ["all"]) -multi_instance = True -ldap = not_relevant -sso = not_relevant +multi_instance = true + # FIXME: replace with an **estimate** minimum disk and RAM requirements. e.g. 20M, 400M, 1G... You may have have a look at CI results disk = "50M" ram.build = "50M" @@ -46,30 +37,14 @@ ram.runtime = "50M" [install.path] # this is a generic question - ask strings are automatically handled by YunoHost's core type = "path" - default = "/example" + default = "/syncserver" + [install.init_main_permission] # this is a generic question - ask strings are automatically handled by YunoHost's core # This won't be saved as setting and will instead be used to initialize the SSOwat permission type = "group" default = "visitors" - [install.language] - ask.en = "Choose the application language" - ask.fr = "Choisissez la langue de l'application" - type = "select" - choices = ['en'] - default = "en" - - [install.admin] - # this is a generic question - ask strings are automatically handled by YunoHost's core - type = "user" - [install.password] - # this is a generic question - ask strings are automatically handled by YunoHost's core - # Note that user-provided passwords questions are not automatically saved as setting - help.en = "Use the help field to add an information for the admin about this question." - help.fr = "Utilisez le champ aide pour ajouter une information à l'intention de l'administrateur à propos de cette question." - type = "password" - [resources] # See the packaging documentation for the full set # of explanation regarding the behavior and properties for each of those @@ -81,26 +56,27 @@ ram.runtime = "50M" # You can also define other assets than "main" and add --source_id="foobar" in the previous command url = "https://github.com/mozilla-services/syncstorage-rs/archive/refs/tags/0.13.6.tar.gz" sha256 = "ddf8169a0a9bd06db10bf032a2b9af7ea4e5816251e82d9e7627f87eba1f27bd" - - # These infos are used by https://github.com/YunoHost/apps/blob/master/tools/autoupdate_app_sources/autoupdate_app_sources.py - # to auto-update the previous asset urls and sha256sum + manifest version - # assuming the upstream's code repo is on github and relies on tags or releases - # See the 'sources' resource documentation for more details - autoupdate.strategy = "latest_github_tag" - [resources.system_user] + # This will provision/deprovision a unix system user - [resources.install_dir] + [resources.system_user] + # This will create/remove the install dir as /var/www/$app # and store the corresponding setting $install_dir - [resources.permissions] + [resources.install_dir] + # This will configure SSOwat permission for $domain/$path/ # The initial allowed group of user is configured via the init_main_permission question (public=visitors, private=all_users) + [resources.permissions] main.url = "/" - [resources.ports] # This will pick a random port for reverse-proxying and store it as the $port setting + [resources.ports] + + [resources.apt] + packages = "mariadb-server default-libmysqlclient-dev cmake python3-dev python3-pip python3-setuptools python3-wheel build-essential libssl-dev libffi-dev libcurl4" + + # This will automatically provision/deprovison a database and store the corresponding credentials in settings $db_user, $db_name, $db_pwd [resources.database] - # This will automatically provision/deprovison a database and store the corresponding credentials in settings $db_user, $db_name, $db_pwd type = "mysql" \ No newline at end of file diff --git a/scripts/_common.sh b/scripts/_common.sh index 944a65e..1021f79 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -4,6 +4,8 @@ # COMMON VARIABLES #================================================= +GO_VERSION="1.20" + #================================================= # PERSONAL HELPERS #================================================= diff --git a/scripts/_common_sh b/scripts/_common_sh deleted file mode 100644 index 69c72c9..0000000 --- a/scripts/_common_sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -#================================================= -# COMMON VARIABLES -#================================================= - -#================================================= -# PERSONAL HELPERS -#================================================= - -#================================================= -# EXPERIMENTAL HELPERS -#================================================= - -#================================================= -# FUTURE OFFICIAL HELPERS -#================================================= \ No newline at end of file diff --git a/scripts/install b/scripts/install index 3293e53..60c1ff2 100644 --- a/scripts/install +++ b/scripts/install @@ -9,32 +9,21 @@ # IMPORT GENERIC HELPERS #================================================= source _common.sh +source ynh_install_go source /usr/share/yunohost/helpers -# Install parameters are automatically saved as settings -# -# Settings are automatically loaded as bash variables -# in every app script context, therefore typically these will exist: -# - $domain -# - $path -# - $language -# ... etc -# -# Resources defined in the manifest are provisioned prior to this script -# and corresponding settings are also available, such as: -# - $install_dir -# - $port -# - $db_name -# ... +salt=$(ynh_string_random --length=40) -# -# $app is the app id (i.e. 'example' for first install, -# or 'example__2', '__3', ... for multi-instance installs) -# #================================================= # INSTALL DEPENDENCIES #================================================= ynh_script_progression --message="Installing dependencies..." --weight=10 +ynh_exec_warn_less ynh_install_go --go_version=$GO_VERSION +export GOPATH="$install_dir/build/go" +export GOCACHE="$install_dir/build/.cache" + +ynh_exec_warn_less ynh_exec_as $app RUSTUP_HOME="$install_dir"/.rustup CARGO_HOME="$install_dir"/.cargo bash -c 'curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -q -y' +export PATH="$install_dir/.cargo/bin:$PATH" #================================================= # APP "BUILD" (DEPLOYING SOURCES, VENV, COMPILING ETC) @@ -47,13 +36,40 @@ ynh_script_progression --message="Setting up source files..." --weight=1 ### downloaded from an upstream source, like a git repository. ### `ynh_setup_source` use the file conf/app.src # Download, check integrity, uncompress and patch the source from app.src -ynh_setup_source --dest_dir="$install_dir" +ynh_setup_source --dest_dir="$install_dir/build" # $install_dir will automatically be initialized with some decent # permission by default ... however, you may need to recursively reapply # ownership to all files such as after the ynh_setup_source step chown -R $app:www-data "$install_dir" +#================================================= +# BUILD +#================================================= +ynh_script_progression --message="Building the sources (it will take some time)..." --weight=10 + +ynh_exec_as $app virtualenv --python=python3 --system-site-packages "${install_dir}/venv" +( + set +o nounset + source "${install_dir}/venv/bin/activate" + set -o nounset + + ynh_use_go + + ynh_exec_warn_less ynh_exec_as $app env PATH=$PATH pip install --upgrade pip + ynh_exec_warn_less ynh_exec_as $app env PATH=$PATH pip install --upgrade setuptools + ynh_exec_warn_less ynh_exec_as $app env PATH=$PATH GOPATH=$GOPATH GOCACHE=$GOCACHE CGO_ENABLED=1 pip install -r requirements.txt + pushd $install_dir/build + ynh_exec_warn_less ynh_exec_as $app env PATH=$PATH GOPATH=$GOPATH GOCACHE=$GOCACHE CGO_ENABLED=1 cargo install --path ../syncserver --locked --root /app + ynh_exec_warn_less ynh_exec_as $app env PATH=$PATH GOPATH=$GOPATH GOCACHE=$GOCACHE CGO_ENABLED=1 cargo install --path ../syncserver --locked --root /app --bin purge_ttl + popd +) + +#ynh_secure_remove --file="$install_dir/.cargo" +#ynh_secure_remove --file="$install_dir/.rustup" +#ynh_secure_remove --file="$install_dir/.cache" +#ynh_secure_remove --file="$install_dir/.local" +#ynh_secure_remove --file="$install_dir/build" #================================================= # SYSTEM CONFIGURATION @@ -65,9 +81,9 @@ ynh_add_nginx_config # Create a dedicated systemd config ynh_add_systemd_config - yunohost service add $app --description="Firefox Sync Server (Rust)" --log="/var/log/$app/$app.log" +yunohost service add $app --description="Firefox Sync Server (Rust)" --log="/var/log/$app/$app.log" - # Use logrotate to manage application logfile(s) +# Use logrotate to manage application logfile(s) ynh_use_logrotate diff --git a/scripts/ynh_install_go b/scripts/ynh_install_go new file mode 100644 index 0000000..f801501 --- /dev/null +++ b/scripts/ynh_install_go @@ -0,0 +1,247 @@ +#!/bin/bash + +ynh_go_try_bash_extension() { + if [ -x src/configure ]; then + src/configure && make -C src || { + ynh_print_info --message="Optional bash extension failed to build, but things will still work normally." + } + fi +} + +goenv_install_dir="/opt/goenv" +go_version_path="$goenv_install_dir/versions" +# goenv_ROOT is the directory of goenv, it needs to be loaded as a environment variable. +export GOENV_ROOT="$goenv_install_dir" + +# Load the version of Go for an app, and set variables. +# +# ynh_use_go has to be used in any app scripts before using Go for the first time. +# This helper will provide alias and variables to use in your scripts. +# +# To use gem or Go, use the alias `ynh_gem` and `ynh_go` +# Those alias will use the correct version installed for the app +# For example: use `ynh_gem install` instead of `gem install` +# +# With `sudo` or `ynh_exec_as`, use instead the fallback variables `$ynh_gem` and `$ynh_go` +# And propagate $PATH to sudo with $ynh_go_load_path +# Exemple: `ynh_exec_as $app $ynh_go_load_path $ynh_gem install` +# +# $PATH contains the path of the requested version of Go. +# However, $PATH is duplicated into $go_path to outlast any manipulation of $PATH +# You can use the variable `$ynh_go_load_path` to quickly load your Go version +# in $PATH for an usage into a separate script. +# Exemple: $ynh_go_load_path $install_dir/script_that_use_gem.sh` +# +# +# Finally, to start a Go service with the correct version, 2 solutions +# Either the app is dependent of Go or gem, but does not called it directly. +# In such situation, you need to load PATH +# `Environment="__YNH_GO_LOAD_PATH__"` +# `ExecStart=__FINALPATH__/my_app` +# You will replace __YNH_GO_LOAD_PATH__ with $ynh_go_load_path +# +# Or Go start the app directly, then you don't need to load the PATH variable +# `ExecStart=__YNH_GO__ my_app run` +# You will replace __YNH_GO__ with $ynh_go +# +# +# one other variable is also available +# - $go_path: The absolute path to Go binaries for the chosen version. +# +# usage: ynh_use_go +# +# Requires YunoHost version 3.2.2 or higher. +ynh_use_go () { + go_version=$(ynh_app_setting_get --app=$app --key=go_version) + + # Get the absolute path of this version of Go + go_path="$go_version_path/$go_version/bin" + + # Allow alias to be used into bash script + shopt -s expand_aliases + + # Create an alias for the specific version of Go and a variable as fallback + ynh_go="$go_path/go" + alias ynh_go="$ynh_go" + + # Load the path of this version of Go in $PATH + if [[ :$PATH: != *":$go_path"* ]]; then + PATH="$go_path:$PATH" + fi + # Create an alias to easily load the PATH + ynh_go_load_path="PATH=$PATH" + + # Sets the local application-specific Go version + pushd $install_dir + $goenv_install_dir/bin/goenv local $go_version + popd +} + +# Install a specific version of Go +# +# ynh_install_go will install the version of Go provided as argument by using goenv. +# +# This helper creates a /etc/profile.d/goenv.sh that configures PATH environment for goenv +# for every LOGIN user, hence your user must have a defined shell (as opposed to /usr/sbin/nologin) +# +# Don't forget to execute go-dependent command in a login environment +# (e.g. sudo --login option) +# When not possible (e.g. in systemd service definition), please use direct path +# to goenv shims (e.g. $goenv_ROOT/shims/bundle) +# +# usage: ynh_install_go --go_version=go_version +# | arg: -v, --go_version= - Version of go to install. +# +# Requires YunoHost version 3.2.2 or higher. +ynh_install_go () { + # Declare an array to define the options of this helper. + local legacy_args=v + local -A args_array=( [v]=go_version= ) + local go_version + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Move an existing Go binary, to avoid to block goenv + test -x /usr/bin/go && mv /usr/bin/go /usr/bin/go_goenv + + # Install or update goenv + goenv="$(command -v goenv $goenv_install_dir/bin/goenv | head -1)" + if [ -n "$goenv" ]; then + ynh_print_info --message="goenv already seems installed in \`$goenv'." + pushd "${goenv%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/syndbg/goenv.git"; then + echo "Trying to update with git..." + git pull -q --tags origin master + cd .. + ynh_go_try_bash_extension + fi + popd + else + ynh_print_info --message="Installing goenv with git..." + mkdir -p $goenv_install_dir + pushd $goenv_install_dir + git init -q + git remote add -f -t master origin https://github.com/syndbg/goenv.git > /dev/null 2>&1 + git checkout -q -b master origin/master + ynh_go_try_bash_extension + goenv=$goenv_install_dir/bin/goenv + popd + fi + + goenv_latest="$(command -v "$goenv_install_dir"/plugins/*/bin/goenv-latest goenv-latest | head -1)" + if [ -n "$goenv_latest" ]; then + ynh_print_info --message="\`goenv latest' command already available in \`$goenv_latest'." + pushd "${goenv_latest%/*/*}" + if git remote -v 2>/dev/null | grep "https://github.com/momo-lab/xxenv-latest.git"; then + ynh_print_info --message="Trying to update xxenv-latest with git..." + git pull -q origin master + fi + popd + else + ynh_print_info --message="Installing xxenv-latest with git..." + mkdir -p "${goenv_install_dir}/plugins" + git clone -q https://github.com/momo-lab/xxenv-latest.git "${goenv_install_dir}/plugins/xxenv-latest" + fi + + # Enable caching + mkdir -p "${goenv_install_dir}/cache" + + # Create shims directory if needed + mkdir -p "${goenv_install_dir}/shims" + + # Restore /usr/local/bin in PATH + PATH=$CLEAR_PATH + + # And replace the old Go binary + test -x /usr/bin/go_goenv && mv /usr/bin/go_goenv /usr/bin/go + + # Install the requested version of Go + local final_go_version=$(goenv latest --print $go_version) + ynh_print_info --message="Installation of Go-$final_go_version" + goenv install --skip-existing $final_go_version + + # Store go_version into the config of this app + ynh_app_setting_set --app=$YNH_APP_INSTANCE_NAME --key=go_version --value=$final_go_version + + # Cleanup Go versions + ynh_cleanup_go + + # Set environment for Go users + echo "#goenv +export GOENV_ROOT=$goenv_install_dir +export PATH=\"$goenv_install_dir/bin:$PATH\" +eval \"\$(goenv init -)\" +#goenv" > /etc/profile.d/goenv.sh + + # Load the environment + eval "$(goenv init -)" +} + +# Remove the version of Go used by the app. +# +# This helper will also cleanup Go versions +# +# usage: ynh_remove_go +ynh_remove_go () { + local go_version=$(ynh_app_setting_get --app=$YNH_APP_INSTANCE_NAME --key=go_version) + + # Load goenv path in PATH + local CLEAR_PATH="$goenv_install_dir/bin:$PATH" + + # Remove /usr/local/bin in PATH in case of Go prior installation + PATH=$(echo $CLEAR_PATH | sed 's@/usr/local/bin:@@') + + # Remove the line for this app + ynh_app_setting_delete --app=$YNH_APP_INSTANCE_NAME --key=go_version + + # Cleanup Go versions + ynh_cleanup_go +} + +# Remove no more needed versions of Go used by the app. +# +# This helper will check what Go version are no more required, +# and uninstall them +# If no app uses Go, goenv will be also removed. +# +# usage: ynh_cleanup_go +ynh_cleanup_go () { + + # List required Go versions + local installed_apps=$(yunohost app list --output-as json --quiet | jq -r .apps[].id) + local required_go_versions="" + for installed_app in $installed_apps + do + local installed_app_go_version=$(ynh_app_setting_get --app=$installed_app --key="go_version") + if [[ $installed_app_go_version ]] + then + required_go_versions="${installed_app_go_version}\n${required_go_versions}" + fi + done + + # Remove no more needed Go versions + local installed_go_versions=$(goenv versions --bare --skip-aliases | grep -Ev '/') + for installed_go_version in $installed_go_versions + do + if ! `echo ${required_go_versions} | grep "${installed_go_version}" 1>/dev/null 2>&1` + then + ynh_print_info --message="Removing of Go-$installed_go_version" + $goenv_install_dir/bin/goenv uninstall --force $installed_go_version + fi + done + + # If none Go version is required + if [[ ! $required_go_versions ]] + then + # Remove goenv environment configuration + ynh_print_info --message="Removing of goenv" + ynh_secure_remove --file="$goenv_install_dir" + ynh_secure_remove --file="/etc/profile.d/goenv.sh" + fi +}