gitlab_ynh/scripts/upgrade

#!/bin/bash

#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================

source _common.sh
source /usr/share/yunohost/helpers

#=================================================
# LOAD SETTINGS
#=================================================

app=$YNH_APP_INSTANCE_NAME

# Retrieve app settings
domain=$(ynh_app_setting_get --app="$app" --key=domain)
path_url=$(ynh_app_setting_get --app="$app" --key=path)
final_path=$(ynh_app_setting_get --app=$app --key=final_path)
config_path=$(ynh_app_setting_get --app=$app --key=config_path)
port=$(ynh_app_setting_get --app="$app" --key=web_port)
puma_port=$(ynh_app_setting_get --app="$app" --key=puma_port)
sidekiq_port=$(ynh_app_setting_get --app="$app" --key=sidekiq_port)
architecture=$(ynh_app_setting_get --app="$app" --key=architecture)
puma_worker_processes=$(ynh_app_setting_get --app="$app" --key=puma_workers)
puma_min_threads=$(ynh_app_setting_get --app="$app" --key=puma_min_threads)
puma_max_threads=$(ynh_app_setting_get --app="$app" --key=puma_max_threads)
client_max_body_size=$(ynh_app_setting_get --app="$app" --key=client_max_body_size)
overwrite_nginx=$(ynh_app_setting_get --app="$app" --key=overwrite_nginx)

#=================================================
# CHECK VERSION
#=================================================

upgrade_type=$(ynh_check_app_version_changed)

#=================================================
# ENSURE DOWNWARD COMPATIBILITY
#=================================================

# If final_path doesn't exist, create it
if [ -z "$final_path" ]; then
	final_path=/opt/$app
	ynh_app_setting_set --app=$app --key=final_path --value=$final_path
fi

# If config_path doesn't exist, create it
if [ -z "$config_path" ]; then
	config_path=/etc/$app
	ynh_app_setting_set --app=$app --key=config_path --value=$config_path
fi

if [ -z "$puma_max_threads" ] || [ -z "$puma_min_threads" ]; then
	# If the server has less than 2GB of RAM
	if [ $(ynh_get_ram --total --ignore_swap) -lt 2000 ]; then
		puma_min_threads=1
		puma_max_threads=1
	else
		puma_min_threads=2
		puma_max_threads=4
	fi

	ynh_app_setting_set --app=$app --key=puma_max_threads --value=$puma_max_threads
	ynh_app_setting_set --app=$app --key=puma_min_threads --value=$puma_min_threads

	ynh_app_setting_delete --app=$app --key=unicorn_worker_processes
fi

# If architecture doesn't exist, create it
if [ -z "$architecture" ]; then
	if [ -n "$(uname -m | grep arm64)" ] || [ -n "$(uname -m | grep aarch64)" ]; then
		architecture="arm64"
	elif [ -n "$(uname -m | grep x86_64)" ]; then
		architecture="x86-64"
	elif [ -n "$(uname -m | grep 86)" ]; then
		ynh_die "Gitlab is not compatible with x86 architecture"
	elif [ -n "$(uname -m | grep arm)" ]; then
		architecture="arm"
	else
		ynh_die --message="Unable to detect your achitecture, please open a bug describing \
			your hardware and the result of the command \"uname -m\"." 1
	fi
	ynh_app_setting_set --app=$app --key=architecture --value=$architecture
fi

# If client_max_body_size doesn't exist, create it
if [ -z "$client_max_body_size" ]; then
	client_max_body_size="250m"
	ynh_app_setting_set --app=$app --key=client_max_body_size --value=$client_max_body_size
fi

# If overwrite_nginx doesn't exist, create it
if [ -z "$overwrite_nginx" ]; then
	overwrite_nginx=1
	ynh_app_setting_set --app=$app --key=overwrite_nginx --value=$overwrite_nginx
fi

# If domain doesn't exist, retrieve it
if [ -z "$domain" ]; then
	domain=$(grep "external_url" "/etc/gitlab/gitlab.rb" | cut -d'/' -f3) # retrieve $domain from conf file
	if [ ${domain: -1} == "'" ]; then # if the last char of $domain is ' remove it
		domain=${domain:0:-1}
	fi
	ynh_app_setting_set --app=$app --key=domain --value=$domain
fi

# If path_url doesn't exist, retrieve it
if [ -z "$path_url" ]; then
	path_url=$(grep "location" "/etc/nginx/conf.d/${domain}.d/gitlab.conf" | cut -d' ' -f2)
	path_url=$(ynh_normalize_url_path $path_url)
	ynh_app_setting_set --app=$app --key=path --value=path_url
fi

# If port doesn't exist, retrieve it
if [ -z "$port" ]; then
	port=$(grep -F "nginx['listen_port']" "/etc/gitlab/gitlab.rb" | cut -d' ' -f3)
	ynh_app_setting_set --app=$app --key=web_port --value=$port
fi

# If port doesn't exist, retrieve it
if [ -z "$puma_port" ]; then
	if [ -z "$(ynh_app_setting_get --app="$app" --key=unicorn_port)" ]; then
		puma_port=$(grep -F "unicorn['port']" "/etc/gitlab/gitlab.rb" | cut -d' ' -f3)
	else
		puma_port=$(ynh_app_setting_get --app="$app" --key=unicorn_port)
	fi
	ynh_app_setting_set --app=$app --key=puma_port --value=$puma_port
	ynh_app_setting_delete --app=$app --key=unicorn_port
fi

if [ -z "$sidekiq_port" ]; then
	sidekiq_port=$(ynh_find_port $(($puma_port + 1)))
	ynh_app_setting_set --app=$app --key=sidekiq_port --value=$sidekiq_port
fi

# if this source file exist, remove it
if [ -e "/etc/apt/sources.list.d/gitlab-ce.list" ]; then
	ynh_secure_remove --file="/etc/apt/sources.list.d/gitlab-ce.list"
fi

#=================================================
# BACKUP BEFORE UPGRADE THEN ACTIVE TRAP
#=================================================
ynh_script_progression --message="Backing up the app before upgrading (may take a while)..." --weight=10

# Backup the current version of the app
ynh_backup_before_upgrade
ynh_clean_setup () {
	ynh_exec_warn_less ynh_secure_remove --file="$tempdir"
	
	ynh_clean_check_starting

	# restore it if the upgrade fails
	ynh_restore_upgradebackup
}
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors

#=================================================
# STANDARD UPGRADE STEPS
#=================================================
# INSTALL DEPENDENCIES
#=================================================
ynh_script_progression --message="Installing dependencies..." --weight=5

ynh_install_app_dependencies $pkg_dependencies

#=================================================
# DEFINE THE NUMBER OF WORKERS USED
#=================================================

total_memory=$(ynh_get_ram --total)

if [ $total_memory -lt 4096 ]; then
	#https://docs.gitlab.com/omnibus/settings/puma.html#running-in-memory-constrained-environments
	puma_worker_processes=0
else
	#https://docs.gitlab.com/ce/install/requirements.html#puma-workers
	puma_worker_processes=$(( $(nproc) > 2 ? $(($(nproc) - 1)) : 2 ))
fi

ynh_app_setting_set --app=$app --key=puma_workers --value=$puma_worker_processes

#=================================================
# ADD SWAP IF NEEDED
#=================================================

total_swap=$(ynh_get_ram --total --only_swap)
swap_needed=0

# https://docs.gitlab.com/ce/install/requirements.html#memory
# Need at least 2Go of swap
if [ $total_swap -lt 2048 ]; then
	swap_needed=$((2048 - total_swap))
fi

if [ $swap_needed -gt 0 ]; then
	ynh_script_progression --message="Adding $swap_needed Mo to swap..."
	if ! ynh_add_swap --size=$swap_needed; then
		ynh_print_warn --message="Please add $swap_needed Mo to swap to make Gitlab work properly"
	fi
fi

#=================================================
# CHECK IF KERNEL IS READ-ONLY
#=================================================

modify_kernel_parameters="true"

for value_to_check in "kernel.shmall" "kernel.shmmax" "kernel.sem" "net.core.somaxconn"
do
	if ! ynh_exec_fully_quiet sysctl --write $value_to_check="$(sysctl --value $value_to_check)"; then
		modify_kernel_parameters="false"
		break
	fi
done

# For gitlab rb

mkdir -p $config_path
ssh_port=$(grep -P "Port\s+\d+" /etc/ssh/sshd_config | grep -P -o "\d+")

generated_external_url="https://$domain${path_url%/}"

#=================================================
# DOWNLOAD, CHECK AND UNPACK SOURCE
#=================================================

if [ "$upgrade_type" == "UPGRADE_APP" ]
then
	ynh_script_progression --message="Setting up source files..." --weight=50

	# To avoid the automatic backup, already performed by YunoHost: https://docs.gitlab.com/omnibus/update/#updating-methods
	touch $config_path/skip-auto-backup

	current_version=$(grep gitlab-ce /opt/gitlab/version-manifest.txt | cut -d' ' -f2)

	# Load the last available version
	source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.last.sh
	last_version=$gitlab_version

	source_current_major_version () {
		if [ -e "$YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.first.sh" ]; then
			source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.first.sh
		elif [ -e "$YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh" ]; then
			source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh
		# Finish with the last migration if the file doesn't exist
		else
			source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.last.sh
		fi
	}

	# To update GitLab from major version A to B, we have to go to the last minor version
	# of the major version A and then go to the first minor version of the major version B
	# to finally go to the current minor version of the major version B
	# A.last -> B.first -> B.last

	# While the current version is not the last version, do an upgrade
	while [ "$last_version" != "$current_version" ]
	do
		# https://docs.gitlab.com/ee/update/#checking-for-background-migrations-before-upgrading
		if dpkg --compare-versions "$current_version" "ge" "12.9"; then
			checkBackgroundMigration=1
		else
			checkBackgroundMigration=0;
		fi
		if dpkg --compare-versions "$current_version" "ge" "15.1"; then
			checkDBBackgroundMigration=2
		elif dpkg --compare-versions "$current_version" "ge" "14.0"; then
			checkDBBackgroundMigration=1
		else
			checkDBBackgroundMigration=0;
		fi
		if dpkg --compare-versions "$current_version" "ge" "14.7"; then
			checkBatchedBackgroundMigration=1
		else
			checkBatchedBackgroundMigration=0;
		fi

		counter=0
		while [ $checkBackgroundMigration -eq 1 ] || [ $checkDBBackgroundMigration -ge 1 ] || [ $checkBatchedBackgroundMigration -eq 1 ]
		do
			counter=$((counter + 1))
			if [ $counter -gt 1200 ]
			then
				ynh_print_warn --message="Timeout: a background migration runs for at least 20min !"
				break
			fi

			if [ $checkBackgroundMigration -eq 1 ] && gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' | grep -q -w 0
			then
				checkBackgroundMigration=0
			fi

			if [ $checkDBBackgroundMigration -eq 2 ] && gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' | grep -q -w 0
			then
				checkDBBackgroundMigration=0
			elif [ $checkDBBackgroundMigration -eq 1 ] && gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending' | wc -l | grep -q -w 0
			then
				checkDBBackgroundMigration=0
			fi

			if [ $checkBatchedBackgroundMigration -eq 1 ] && echo "select COUNT(*) from batched_background_migrations where status <> 3;" | gitlab-psql -t -A | grep -q -w 0
			then
				checkBatchedBackgroundMigration=0
			fi
			ynh_print_info --message="Wait for the migration in the background to finish"
			sleep 1
		done

		current_major_version=${current_version%%.*}

		source_current_major_version

		# if the version stored in the upgrade.$current_major_version.first.sh file is less than or equal
		# to the current version, and if the version stored in the upgrade.$current_major_version.last.sh file
		# increment the major version to upgrade to the next version
		if dpkg --compare-versions "$gitlab_version" "le" "$current_version"; then
			if [ -e "$YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh" ]; then
				source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh
			else
				source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.last.sh
			fi
			if dpkg --compare-versions "$gitlab_version" "le" "$current_version"; then
				current_major_version=$(($current_major_version + 1))
				source_current_major_version
			fi
		fi

		ynh_add_config --template="$YNH_APP_BASEDIR/conf/$architecture.src.default" --destination="$YNH_APP_BASEDIR/conf/$architecture.src"

		tempdir="$(mktemp -d)"

		ynh_setup_source --dest_dir=$tempdir --source_id=$architecture

		if ! ynh_exec_warn_less dpkg -i $tempdir/$gitlab_filename; then # This command will fail in lxc env
			package_check_action # defined in upgrade.d/upgrade.X.sh
			ynh_exec_warn_less dpkg --configure gitlab-ce
		fi

		ynh_exec_warn_less ynh_secure_remove --file="$tempdir"

		current_version=$(grep gitlab-ce /opt/gitlab/version-manifest.txt | cut -d' ' -f2)

		# Sometimes we need to update the gitlab.rb configuration file in order to migrate to the next version.
		if [ -e "$YNH_APP_BASEDIR/conf/gitlab.$current_major_version.rb" ]; then
			if [ -e "$YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh" ]; then
				source $YNH_APP_BASEDIR/scripts/upgrade.d/upgrade.$current_major_version.last.sh
			fi
			if dpkg --compare-versions "$gitlab_version" "ge" "$current_version"; then
				ynh_add_config --template="$YNH_APP_BASEDIR/conf/gitlab.$current_major_version.rb" --destination="$config_path/gitlab.rb"

				touch "$config_path/gitlab-persistent.rb"
				chown root:root "$config_path/gitlab-persistent.rb"
				chmod 640 "$config_path/gitlab-persistent.rb"

				# During large migrations, the logs are too big to be sent to paste.yunohost.org
				# Send the reconfigure logs in a file, and if the process succeeds, just delete it.
				gitlab-ctl reconfigure > "/tmp/gitlab_upgrade_$current_version.log"
				ynh_secure_remove --file="/tmp/gitlab_upgrade_$current_version.log"
			fi
		fi
	done
fi

#=================================================
# RECONFIGURE GITLAB
#=================================================
ynh_script_progression --message="Reconfigure GitLab..." --weight=13

ynh_add_config --template="$YNH_APP_BASEDIR/conf/gitlab.rb" --destination="$config_path/gitlab.rb"

touch "$config_path/gitlab-persistent.rb"
chown root:root "$config_path/gitlab-persistent.rb"
chmod 640 "$config_path/gitlab-persistent.rb"

ynh_exec_warn_less gitlab-ctl reconfigure

# Allow ssh for git
usermod -a -G "ssh.app" "git"

#=================================================
# NGINX CONFIGURATION
#=================================================

# Overwrite the nginx configuration only if it's allowed
if [ $overwrite_nginx -eq 1 ]
then
	ynh_script_progression --message="Configuring NGINX web server..." --weight=2
	# Create a dedicated nginx config
	ynh_add_nginx_config client_max_body_size
fi

#=================================================
# GENERIC FINALIZATION
#=================================================
# ADVERTISE SERVICE IN ADMIN PANEL
#=================================================

yunohost service add "gitlab-runsvdir" --log "/var/log/$app/gitlab-rails/application.log" "/var/log/$app/gitlab-rails/api_json.log" "/var/log/$app/gitlab-rails/production.log" "/var/log/$app/gitlab-rails/production_json.log" "/var/log/$app/gitlab-rails/sidekiq.log" "/var/log/$app/puma/puma_stderr.log" "/var/log/$app/puma/current" "/var/log/$app/alertmanager/current" "/var/log/$app/gitaly/current" "/var/log/$app/gitlab-monitor/current" "/var/log/$app/gitlab-shell/gitlab-shell.log" "/var/log/$app/gitlab-workhorse/current" "/var/log/$app/logrotate/current" "/var/log/$app/nginx/current" "/var/log/$app/nginx/access.log" "/var/log/$app/nginx/error.log" "/var/log/$app/nginx/gitlab_access.log" "/var/log/$app/nginx/gitlab_error.log" "/var/log/$app/node-exporter/current" "/var/log/$app/postgres-exporter/current" "/var/log/$app/postgresql/current" "/var/log/$app/prometheus/current" "/var/log/$app/redis/current" "/var/log/$app/redis-exporter/current"

#=================================================
# WAITING GITLAB
#=================================================

if [ "$upgrade_type" == "UPGRADE_APP" ]
then
	ynh_script_progression --message="Restarting GitLab..." --weight=15

	ynh_systemd_action --action=restart --service_name="gitlab-runsvdir" --log_path="/var/log/$app/puma/current" --line_match="Listening on http://127.0.0.1:$puma_port" --timeout=300
fi

#=================================================
# RELOAD NGINX
#=================================================
ynh_script_progression --message="Reloading NGINX web server..." --weight=1

ynh_systemd_action --action=reload --service_name=nginx

#=================================================
# END OF SCRIPT
#=================================================

ynh_script_progression --message="Upgrade of GitLab completed" --last