From 1b2138fb65299bbdef63006dbbf980ae2b68fbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Sat, 21 Jan 2023 16:03:46 +0100 Subject: [PATCH] Allow an automagic Extensions url update --- .github/workflows/update_extensions.py | 23 ++++++++++++++++++---- scripts/_common.sh | 27 ++++++++++++++++++++++++++ scripts/install | 12 ++++++------ scripts/upgrade | 12 ++++++------ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/.github/workflows/update_extensions.py b/.github/workflows/update_extensions.py index 9f4de1c..850dd28 100755 --- a/.github/workflows/update_extensions.py +++ b/.github/workflows/update_extensions.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 +""" +Download extensions for the current mediawiki version, and update the conf files. +""" -from pathlib import Path +import argparse from typing import List, Optional import hashlib import json @@ -25,12 +28,13 @@ EXTENSIONS = { def sha256sum_of_url(url: str) -> str: """Compute checksum without saving the file""" checksum = hashlib.sha256() - for chunk in requests.get(url, stream=True).iter_content(): + for chunk in requests.get(url, stream=True, timeout=10).iter_content(): checksum.update(chunk) return checksum.hexdigest() def generate_ext_source(asset_url: str, src_filename: str) -> None: + """Generate the conf file""" with open(f"conf/{src_filename}", "w", encoding="utf-8") as output: output.write(textwrap.dedent(f"""\ SOURCE_URL={asset_url} @@ -50,6 +54,7 @@ def get_all_extensions() -> List[str]: class MyHTMLParser(HTMLParser): links = [] + def handle_starttag(self, tag, attrs): if tag == "a": for name, value in attrs: @@ -61,6 +66,7 @@ def get_all_extensions() -> List[str]: return parser.links def find_valid_ext(all_exts: List[str], name: str, max_version: version.Version) -> Optional[str]: + """Find the valid extensions for the current mediawiki version""" def version_of(ext): try: return version.parse(ext.split("-")[1].replace("_", ".").replace("REL", "")) @@ -70,7 +76,7 @@ def find_valid_ext(all_exts: List[str], name: str, max_version: version.Version) found_exts = [ ext for ext in all_exts - if ext.startswith(name) and version_of(ext) <= max_version + if ext.startswith(name) and version_of(ext) <= max_version ] return max(found_exts, key=version_of) if found_exts else None @@ -83,7 +89,16 @@ def main(): all_extensions = get_all_extensions() - for file, name in EXTENSIONS.items(): + parser = argparse.ArgumentParser() + parser.add_argument('extension_file', nargs='?') + args = parser.parse_args() + + if args.extension_file: + extensions = {args.extension_file: EXTENSIONS[args.extension_file]} + else: + extensions = EXTENSIONS + + for file, name in extensions.items(): print(f'Updating source file for {name}') ext = find_valid_ext(all_extensions, name, mediawiki_version) if ext is None: diff --git a/scripts/_common.sh b/scripts/_common.sh index fc8dbe7..4e1ac8d 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -11,6 +11,33 @@ pkg_dependencies="diffutils imagemagick acl" # PERSONAL HELPERS #================================================= +__setup_extension_or_update() { + # The idea is to try to download the extension, but if it fails, let's call + # the update_extensions.py script, and retry. + # It should be a nice workaround for issues like: + # https://github.com/YunoHost-Apps/mediawiki_ynh/issues/91 + + # Same args as ynh_setup_source + # Won't use "$@" because we need the source_id + local -A args_array=([d]=dest_dir= [s]=source_id= [k]=keep= [r]=full_replace=) + local dest_dir + local source_id + local keep + local full_replace + # Manage arguments with getopts + ynh_handle_getopts_args "$@" + + setup_source_args=( + --dest_dir="$dest_dir" --source_id="$source_id" + --keep="$keep" --full_replace="$full_replace" + ) + + ynh_setup_source "${setup_source_args[@]}" || { + ./.github/workflows/update_extensions.py "$source_id" + ynh_setup_source "${setup_source_args[@]}" + } +} + #================================================= # EXPERIMENTAL HELPERS #================================================= diff --git a/scripts/install b/scripts/install index 0243e86..a5ea0ed 100644 --- a/scripts/install +++ b/scripts/install @@ -91,14 +91,14 @@ ynh_script_progression --message="Setting up source files..." --weight=4 ynh_app_setting_set --app=$app --key=final_path --value=$final_path # Download, check integrity, uncompress and patch the source from app.src ynh_setup_source --dest_dir="$final_path" -ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_authentication2" -ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_provider" -ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="pluggable_auth" +__setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_authentication2" +__setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_provider" +__setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="pluggable_auth" # Note(decentral1se): Disabled and unused for now ... -# ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_groups" -# ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_userinfo" -# ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_authorization" +# __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_groups" +# __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_userinfo" +# __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_authorization" chmod 750 "$final_path" chmod -R o-rwx "$final_path" diff --git a/scripts/upgrade b/scripts/upgrade index f15f10d..978e641 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -115,14 +115,14 @@ then # Download, check integrity, uncompress and patch the source from app.src ynh_setup_source --dest_dir="$final_path" - ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_authentication2" - ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_provider" - ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="pluggable_auth" + __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_authentication2" + __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_provider" + __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="pluggable_auth" # Note(decentral1se): Disabled and unused for now ... - # ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_groups" - # ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_userinfo" - # ynh_setup_source --dest_dir="$final_path/extensions/" --source_id="ldap_authorization" + # __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_groups" + # __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_userinfo" + # __setup_extension_or_update --dest_dir="$final_path/extensions/" --source_id="ldap_authorization" fi