1
0
Fork 0
mirror of https://github.com/YunoHost-Apps/leed_ynh.git synced 2024-09-03 19:26:32 +02:00

New parser for the new changelog

This commit is contained in:
Maniack Crudelis 2019-06-02 19:12:29 +02:00
parent f828d2381d
commit 96e3735961
2 changed files with 356 additions and 105 deletions

View file

@ -1,5 +1,9 @@
#!/bin/bash #!/bin/bash
#=================================================
# PERSONAL HELPERS
#=================================================
#================================================= #=================================================
# BACKUP # BACKUP
#================================================= #=================================================
@ -384,128 +388,366 @@ ynh_maintenance_mode_OFF () {
#================================================= #=================================================
# Download and check integrity of a file from app.src_file # Create a changelog for an app after an upgrade from the file CHANGELOG.md.
# #
# The file conf/app.src_file need to contains: # usage: ynh_send_readme_to_admin [--format=markdown/html/plain] [--output=changelog_file] --changelog=changelog_source]
# | arg: -f --format= - Format in which the changelog will be printed
# markdown: Default format.
# html: Turn urls into html format.
# plain: Plain text changelog
# | arg: -o --output= - Output file for the changelog file (Default ./changelog)
# | arg: -c --changelog= - CHANGELOG.md source (Default ../CHANGELOG.md)
# #
# FILE_URL=Address to download the file # The changelog is printed into the file ./changelog and ./changelog_lite
# FILE_SUM=Control sum ynh_app_changelog () {
# # (Optional) Program to check the integrity (sha256sum, md5sum...)
# # default: sha256
# FILE_SUM_PRG=sha256
# # (Optionnal) Name of the local archive (offline setup support)
# # default: Name of the downloaded file.
# FILENAME=example.deb
#
# usage: ynh_download_file --dest_dir="/destination/directory" [--source_id=myfile]
# | arg: -d, --dest_dir= - Directory where to download the file
# | arg: -s, --source_id= - Name of the source file 'app.src_file' if it isn't '$app'
ynh_download_file () {
# Declare an array to define the options of this helper. # Declare an array to define the options of this helper.
declare -Ar args_array=( [d]=dest_dir= [s]=source_id= ) local legacy_args=foc
local dest_dir declare -Ar args_array=( [f]=format= [o]=output= [c]=changelog= )
local source_id local format
local output
local changelog
# Manage arguments with getopts # Manage arguments with getopts
ynh_handle_getopts_args "$@" ynh_handle_getopts_args "$@"
format=${format:-markdown}
output=${output:-changelog}
changelog=${changelog:-../CHANGELOG.md}
source_id=${source_id:-app} # If the argument is not given, source_id equals "$app" local original_changelog="$changelog"
local temp_changelog="changelog_temp"
local final_changelog="$output"
# Load value from configuration file (see above for a small doc about this file if [ ! -n "$original_changelog" ]
# format)
local src_file="$YNH_CWD/../conf/${source_id}.src_file"
# If the src_file doesn't exist, use the backup path instead, with a "settings" directory
if [ ! -e "$src_file" ]
then then
src_file="$YNH_CWD/../settings/conf/${source_id}.src_file" echo "No changelog available..." > "$final_changelog"
fi echo "No changelog available..." > "${final_changelog}_lite"
local file_url=$(grep 'FILE_URL=' "$src_file" | cut -d= -f2-) return 0
local file_sum=$(grep 'FILE_SUM=' "$src_file" | cut -d= -f2-)
local file_sumprg=$(grep 'FILE_SUM_PRG=' "$src_file" | cut -d= -f2-)
local filename=$(grep 'FILENAME=' "$src_file" | cut -d= -f2-)
# Default value
file_sumprg=${file_sumprg:-sha256sum}
if [ "$filename" = "" ] ; then
filename="$(basename "$file_url")"
fi
local local_src="/opt/yunohost-apps-src/${YNH_APP_ID}/${filename}"
if test -e "$local_src"
then # Use the local source file if it is present
cp $local_src $filename
else # If not, download the source
local out=`wget -nv -O $filename $file_url 2>&1` || ynh_print_err $out
fi fi
# Check the control sum local current_version=$(ynh_read_manifest --manifest="/etc/yunohost/apps/$YNH_APP_INSTANCE_NAME/manifest.json" --manifest_key="version")
echo "${file_sum} ${filename}" | ${file_sumprg} -c --status \ local update_version=$(ynh_read_manifest --manifest="../manifest.json" --manifest_key="version")
|| ynh_die "Corrupt file"
# Create the destination directory, if it's not already. # Get the line of the version to update to into the changelog
mkdir -p "$dest_dir" local update_version_line=$(grep --max-count=1 --line-number "^## $update_version" "$original_changelog" | cut -d':' -f1)
# If there's no entry for this version yet into the changelog
# Get the first available version
if [ -z "$update_version_line" ]
then
update_version_line=$(grep --max-count=1 --line-number "^##" "$original_changelog" | cut -d':' -f1)
fi
# Move the file to its destination # Get the length of the complete changelog.
mv $filename $dest_dir local changelog_length=$(wc --lines "$original_changelog" | awk '{print $1}')
# Cut the file before the version to update to.
tail --lines=$(( $changelog_length - $update_version_line + 1 )) "$original_changelog" > "$temp_changelog"
# Get the length of the troncated changelog.
changelog_length=$(wc --lines "$temp_changelog" | awk '{print $1}')
# Get the line of the current version into the changelog
# Keep only the last line found
local current_version_line=$(grep --line-number "^## $current_version" "$temp_changelog" | cut -d':' -f1 | tail --lines=1)
# If there's no entry for this version into the changelog
# Get the last available version
if [ -z "$current_version_line" ]
then
current_version_line=$(grep --line-number "^##" "$original_changelog" | cut -d':' -f1 | tail --lines=1)
fi
# Cut the file before the current version.
# Then grep the previous version into the changelog to get the line number of the previous version
local previous_version_line=$(tail --lines=$(( $changelog_length - $current_version_line )) \
"$temp_changelog" | grep --max-count=1 --line-number "^##" | cut -d':' -f1)
# If there's no previous version into the changelog
# Go until the end of the changelog
if [ -z "$previous_version_line" ]
then
previous_version_line=$changelog_length
fi
# Cut the file after the previous version to keep only the changelog between the current version and the version to update to.
head --lines=$(( $current_version_line + $previous_version_line - 1 )) "$temp_changelog" | tee "$final_changelog"
if [ "$format" = "html" ]
then
# Replace markdown links by html links
ynh_replace_string --match_string="\[\(.*\)\](\(.*\)))" --replace_string="<a href=\"\2\">\1</a>)" --target_file="$final_changelog"
ynh_replace_string --match_string="\[\(.*\)\](\(.*\))" --replace_string="<a href=\"\2\">\1</a>" --target_file="$final_changelog"
elif [ "$format" = "plain" ]
then
# Change title format.
ynh_replace_string --match_string="^\(##.* \)(\[\(.*\)\](\(.*\)))" --replace_string="\1- \2 \3" --target_file="$final_changelog"
# Change modifications lines format.
ynh_replace_string --match_string="^\([-*]\).*\[\(.*\)\]\(.*\)" --replace_string="\1 \2 \3" --target_file="$final_changelog"
fi
# else markdown. As the file is already in markdown, nothing to do.
# Keep only important changes into the changelog
# Remove all minor changes
sed '/^-/d' "$final_changelog" > "${final_changelog}_lite"
# Remove all blank lines (to keep a clear workspace)
sed --in-place '/^$/d' "${final_changelog}_lite"
# Add a blank line at the end
echo "" >> "${final_changelog}_lite"
# Clean titles if there's no significative changes
local line
local previous_line
while read line <&3
do
if [ -n "$previous_line" ]
then
# Remove the line if it's a title or a blank line, and the previous one was a title as well.
if ( [ "${line:0:1}" = "#" ] || [ ${#line} -eq 0 ] ) && [ "${previous_line:0:1}" = "#" ]
then
ynh_replace_special_string --match_string="${previous_line//[/.}" --replace_string="" --target_file="${final_changelog}_lite"
fi
fi
previous_line="$line"
done 3< "${final_changelog}_lite"
# Remove all blank lines again
sed --in-place '/^$/d' "${final_changelog}_lite"
# Restore changelog format with blank lines
ynh_replace_string --match_string="^##.*" --replace_string="\n\n&\n" --target_file="${final_changelog}_lite"
# Remove the 2 first blank lines
sed --in-place '1,2d' "${final_changelog}_lite"
# Add a blank line at the end
echo "" >> "${final_changelog}_lite"
# If changelog are empty, add an info
if [ $(wc --words "$final_changelog" | awk '{print $1}') -eq 0 ]
then
echo "No changes from the changelog..." > "$final_changelog"
fi
if [ $(wc --words "${final_changelog}_lite" | awk '{print $1}') -eq 0 ]
then
echo "No significative changes from the changelog..." > "${final_changelog}_lite"
fi
} }
#================================================= #=================================================
# Create a changelog for an app after an upgrade. # Check the amount of available RAM
# #
# The changelog is printed into the file ./changelog for the time of the upgrade. # usage: ynh_check_ram [--required=RAM required in Mb] [--no_swap|--only_swap] [--free_ram]
# # | arg: -r, --required= - Amount of RAM required in Mb. The helper will return 0 is there's enough RAM, or 1 otherwise.
# In order to create a changelog, ynh_app_changelog will get info from /etc/yunohost/apps/$app/status.json # If --required isn't set, the helper will print the amount of RAM, in Mb.
# In order to find the current commit use by the app. # | arg: -s, --no_swap - Ignore swap
# The remote repository, and the branch. # | arg: -o, --only_swap - Ignore real RAM, consider only swap.
# The changelog will be only the commits since the current revision. # | arg: -f, --free_ram - Count only free RAM, not the total amount of RAM available.
# ynh_check_ram () {
# Because of the need of those info, ynh_app_changelog works only # Declare an array to define the options of this helper.
# with apps that have been installed from a list. declare -Ar args_array=( [r]=required= [s]=no_swap [o]=only_swap [f]=free_ram )
# local required
# usage: ynh_app_changelog local no_swap
ynh_app_changelog () { local only_swap
get_value_from_settings () # Manage arguments with getopts
{ ynh_handle_getopts_args "$@"
local value="$1" required=${required:-}
# Extract a value from the status.json file of an installed app. no_swap=${no_swap:-0}
only_swap=${only_swap:-0}
grep "$value\": \"" /etc/yunohost/apps/$app/status.json | sed "s/.*$value\": \"\([^\"]*\).*/\1/" local total_ram=$(vmstat --stats --unit M | grep "total memory" | awk '{print $1}')
} local total_swap=$(vmstat --stats --unit M | grep "total swap" | awk '{print $1}')
local total_ram_swap=$(( total_ram + total_swap ))
local current_revision="$(get_value_from_settings revision)" local free_ram=$(vmstat --stats --unit M | grep "free memory" | awk '{print $1}')
local repo="$(get_value_from_settings url)" local free_swap=$(vmstat --stats --unit M | grep "free swap" | awk '{print $1}')
local branch="$(get_value_from_settings branch)" local free_ram_swap=$(( free_ram + free_swap ))
# ynh_app_changelog works only with an app installed from a list.
if [ -z "$current_revision" ] || [ -z "$repo" ] || [ -z "$branch" ] # Use the total amount of ram
local ram=$total_ram_swap
if [ $free_ram -eq 1 ]
then then
ynh_print_warn "Unable to build the changelog..." # Use the total amount of free ram
touch changelog ram=$free_ram_swap
if [ $no_swap -eq 1 ]
then
# Use only the amount of free ram
ram=$free_ram
elif [ $only_swap -eq 1 ]
then
# Use only the amount of free swap
ram=$free_swap
fi
else
if [ $no_swap -eq 1 ]
then
# Use only the amount of free ram
ram=$total_ram
elif [ $only_swap -eq 1 ]
then
# Use only the amount of free swap
ram=$total_swap
fi
fi
if [ -n "$required" ]
then
# Return 1 if the amount of ram isn't enough.
if [ $ram -lt $required ]
then
return 1
else
return 0 return 0
fi fi
# Fetch the history of the repository, without cloning it # If no RAM is required, return the amount of available ram.
mkdir git_history else
(cd git_history echo $ram
ynh_exec_warn_less git init fi
ynh_exec_warn_less git remote add -f origin $repo }
# Get the line of the current commit of the installed app in the history.
local line_to_head=$(git log origin/$branch --pretty=oneline | grep --line-number "$current_revision" | cut -d':' -f1) #=================================================
# Cut the history before the current commit, to keep only newer commits.
# Then use sed to reorganise each lines and have a nice list of commits since the last upgrade. # Define the values to configure php-fpm
# This list is redirected into the file changelog #
git log origin/$branch --pretty=oneline | head --lines=$(($line_to_head-1)) | sed 's/^\([[:alnum:]]*\)\(.*\)/*(\1) -> \2/g' > ../changelog) # usage: ynh_get_scalable_phpfpm --usage=usage --footprint=footprint [--print]
# Remove 'Merge pull request' commits # | arg: -f, --footprint - Memory footprint of the service (low/medium/high).
sed -i '/Merge pull request #[[:digit:]]* from/d' changelog # low - Less than 20Mb of ram by pool.
# As well as conflict resolving commits # medium - Between 20Mb and 40Mb of ram by pool.
sed -i '/Merge branch .* into/d' changelog # high - More than 40Mb of ram by pool.
# Or specify exactly the footprint, the load of the service as Mb by pool instead of having a standard value.
# Get the value of admin_mail_html # To have this value, use the following command and stress the service.
admin_mail_html=$(ynh_app_setting_get $app admin_mail_html) # watch -n0.5 ps -o user,cmd,%cpu,rss -u APP
admin_mail_html="${admin_mail_html:-0}" #
# | arg: -u, --usage - Expected usage of the service (low/medium/high).
# If a html email is required. Apply html to the changelog. # low - Personal usage, behind the sso.
if [ "$admin_mail_html" -eq 1 ] # medium - Low usage, few people or/and publicly accessible.
then # high - High usage, frequently visited website.
sed -in-place "s@\*(\([[:alnum:]]*\)) -> \(.*\)@* __URL_TAG1__\2__URL_TAG2__${repo}/commit/\1__URL_TAG3__@g" changelog #
# | arg: -p, --print - Print the result
#
#
#
# The footprint of the service will be used to defined the maximum footprint we can allow, which is half the maximum RAM.
# So it will be used to defined 'pm.max_children'
# A lower value for the footprint will allow more children for 'pm.max_children'. And so for
# 'pm.start_servers', 'pm.min_spare_servers' and 'pm.max_spare_servers' which are defined from the
# value of 'pm.max_children'
# NOTE: 'pm.max_children' can't exceed 4 times the number of processor's cores.
#
# The usage value will defined the way php will handle the children for the pool.
# A value set as 'low' will set the process manager to 'ondemand'. Children will start only if the
# service is used, otherwise no child will stay alive. This config gives the lower footprint when the
# service is idle. But will use more proc since it has to start a child as soon it's used.
# Set as 'medium', the process manager will be at dynamic. If the service is idle, a number of children
# equal to pm.min_spare_servers will stay alive. So the service can be quick to answer to any request.
# The number of children can grow if needed. The footprint can stay low if the service is idle, but
# not null. The impact on the proc is a little bit less than 'ondemand' as there's always a few
# children already available.
# Set as 'high', the process manager will be set at 'static'. There will be always as many children as
# 'pm.max_children', the footprint is important (but will be set as maximum a quarter of the maximum
# RAM) but the impact on the proc is lower. The service will be quick to answer as there's always many
# children ready to answer.
ynh_get_scalable_phpfpm () {
local legacy_args=ufp
# Declare an array to define the options of this helper.
declare -Ar args_array=( [u]=usage= [f]=footprint= [p]=print )
local usage
local footprint
local print
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
# Set all characters as lowercase
footprint=${footprint,,}
usage=${usage,,}
print=${print:-0}
if [ "$footprint" = "low" ]
then
footprint=20
elif [ "$footprint" = "medium" ]
then
footprint=35
elif [ "$footprint" = "high" ]
then
footprint=50
fi
# Define the way the process manager handle child processes.
if [ "$usage" = "low" ]
then
php_pm=ondemand
elif [ "$usage" = "medium" ]
then
php_pm=dynamic
elif [ "$usage" = "high" ]
then
php_pm=static
else
ynh_die --message="Does not recognize '$usage' as an usage value."
fi
# Get the total of RAM available, except swap.
local max_ram=$(ynh_check_ram --no_swap)
less0() {
# Do not allow value below 1
if [ $1 -le 0 ]
then
echo 1
else
echo $1
fi
}
# Define pm.max_children
# The value of pm.max_children is the total amount of ram divide by 2 and divide again by the footprint of a pool for this app.
# So if php-fpm start the maximum of children, it won't exceed half of the ram.
php_max_children=$(( $max_ram / 2 / $footprint ))
# If process manager is set as static, use half less children.
# Used as static, there's always as many children as the value of pm.max_children
if [ "$php_pm" = "static" ]
then
php_max_children=$(( $php_max_children / 2 ))
fi
php_max_children=$(less0 $php_max_children)
# To not overload the proc, limit the number of children to 4 times the number of cores.
local core_number=$(nproc)
local max_proc=$(( $core_number * 4 ))
if [ $php_max_children -gt $max_proc ]
then
php_max_children=$max_proc
fi
if [ "$php_pm" = "dynamic" ]
then
# Define pm.start_servers, pm.min_spare_servers and pm.max_spare_servers for a dynamic process manager
php_min_spare_servers=$(( $php_max_children / 8 ))
php_min_spare_servers=$(less0 $php_min_spare_servers)
php_max_spare_servers=$(( $php_max_children / 2 ))
php_max_spare_servers=$(less0 $php_max_spare_servers)
php_start_servers=$(( $php_min_spare_servers + ( $php_max_spare_servers - $php_min_spare_servers ) /2 ))
php_start_servers=$(less0 $php_start_servers)
else
php_min_spare_servers=0
php_max_spare_servers=0
php_start_servers=0
fi
if [ $print -eq 1 ]
then
ynh_debug --message="Footprint=${footprint}Mb by pool."
ynh_debug --message="Process manager=$php_pm"
ynh_debug --message="Max RAM=${max_ram}Mb"
if [ "$php_pm" != "static" ]; then
ynh_debug --message="\nMax estimated footprint=$(( $php_max_children * $footprint ))"
ynh_debug --message="Min estimated footprint=$(( $php_min_spare_servers * $footprint ))"
fi
if [ "$php_pm" = "dynamic" ]; then
ynh_debug --message="Estimated average footprint=$(( $php_max_spare_servers * $footprint ))"
elif [ "$php_pm" = "static" ]; then
ynh_debug --message="Estimated footprint=$(( $php_max_children * $footprint ))"
fi
ynh_debug --message="\nRaw php-fpm values:"
ynh_debug --message="pm.max_children = $php_max_children"
if [ "$php_pm" = "dynamic" ]; then
ynh_debug --message="pm.start_servers = $php_start_servers"
ynh_debug --message="pm.min_spare_servers = $php_min_spare_servers"
ynh_debug --message="pm.max_spare_servers = $php_max_spare_servers"
fi
fi fi
} }

View file

@ -253,7 +253,16 @@ ynh_maintenance_mode_OFF
admin_panel="https://$(grep portal_domain /etc/ssowat/conf.json | cut -d'"' -f4)/yunohost/admin/#/apps/$app" admin_panel="https://$(grep portal_domain /etc/ssowat/conf.json | cut -d'"' -f4)/yunohost/admin/#/apps/$app"
# Build the changelog # Build the changelog
ynh_app_changelog || true # Get the value of admin_mail_html
admin_mail_html=$(ynh_app_setting_get $app admin_mail_html)
admin_mail_html="${admin_mail_html:-0}"
# If a html email is required. Apply html to the changelog.
if [ "$admin_mail_html" -eq 1 ]; then
format=html
else
format=plain
fi
ynh_app_changelog --format=$format
echo "You can configure this app easily by using the experimental __URL_TAG1__config-panel feature__URL_TAG2__$admin_panel/config-panel__URL_TAG3__. echo "You can configure this app easily by using the experimental __URL_TAG1__config-panel feature__URL_TAG2__$admin_panel/config-panel__URL_TAG3__.
You can also find some specific actions for this app by using the experimental __URL_TAG1__action feature__URL_TAG2__$admin_panel/actions__URL_TAG3__. You can also find some specific actions for this app by using the experimental __URL_TAG1__action feature__URL_TAG2__$admin_panel/actions__URL_TAG3__.