From 634c468840cde9345375b45f0e5ae5ecc02b7420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Fri, 8 Oct 2021 18:49:20 +0200 Subject: [PATCH] Update update_extensions.py --- update_extensions.py | 96 ++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/update_extensions.py b/update_extensions.py index 0b80d11..2f0bb52 100755 --- a/update_extensions.py +++ b/update_extensions.py @@ -1,20 +1,29 @@ #!/usr/bin/env python3 +""" +This tool updates conf/*.src according to the available extension version at wmflabs.org +""" +import sys import json -from os import replace import urllib.request from html.parser import HTMLParser import subprocess import hashlib -import sys, fileinput +import fileinput +from typing import List -extensions_host_url = 'https://extdist.wmflabs.org/dist/extensions/' +EXTENSIONS_HOST_URL = 'https://extdist.wmflabs.org/dist/extensions/' -def get_all_extensions(): - webpage = urllib.request.urlopen(extensions_host_url).read().decode('utf-8') + +def get_all_extensions() -> List[str]: + """Get all available extensions.""" + with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page: + webpage = page.read().decode('utf-8') class MyHTMLParser(HTMLParser): + """Custom HTMLParser""" links = [] + def handle_starttag(self, tag, attrs): # Only parse the 'anchor' tag. if tag == 'a': @@ -28,89 +37,96 @@ def get_all_extensions(): parser.feed(webpage) return parser.links -def get_mediawiki_ext_version(): - with open('manifest.json') as manifest_json: + +def get_mediawiki_ext_version() -> str: + """Returns the mediawiki version for extensions.""" + with open('manifest.json', encoding='utf-8') as manifest_json: manifest = json.load(manifest_json) mediawiki_version = manifest['version'].split('~')[0] mediawiki_ext_version = '_'.join(mediawiki_version.split('.')[0:2]) return mediawiki_ext_version -def get_extensions_for_version(extensions, version): - exts = [ ext for ext in extensions if version in ext ] +def get_extensions_for_version(extensions: List[str], version: str) -> List[str]: + """Returns available extensions compatible with mediawiki version.""" + exts = [ext for ext in extensions if version in ext] return exts - ############################################################################### -def sha256sum(filename): +def sha256sum(filename: str) -> str: + """Calculate the sha256 of a file.""" sha256_hash = hashlib.sha256() - with open(filename,"rb") as f: + with open(filename, "rb") as file: # Read and update hash string value in blocks of 4K - for byte_block in iter(lambda: f.read(4096),b""): + for byte_block in iter(lambda: file.read(4096), b""): sha256_hash.update(byte_block) return sha256_hash.hexdigest() -def replace_line_startingwith(file, startingwith, new_line): +def replace_line_startingwith(file: str, startingwith: str, new_line: str): + """""" for line in fileinput.input(file, inplace=1): if line.startswith(startingwith): line = new_line + '\n' sys.stdout.write(line) - pass -def update_source_file(srcfile, url, shasum): + +def update_source_file(srcfile: str, url: str): filename = url.rsplit('/', 1)[1] urllib.request.urlretrieve(url, filename) - hash = sha256sum(filename) + hashsum = sha256sum(filename) - replace_line_startingwith(srcfile, 'SOURCE_URL=', 'SOURCE_URL={}'.format(url)) - replace_line_startingwith(srcfile, 'SOURCE_SUM=', 'SOURCE_SUM={}'.format(hash)) + replace_line_startingwith(srcfile, 'SOURCE_URL=', f'SOURCE_URL={url}') + replace_line_startingwith(srcfile, 'SOURCE_SUM=', f'SOURCE_SUM={hashsum}') replace_line_startingwith(srcfile, 'SOURCE_SUM_PRG=', 'SOURCE_SUM_PRG=sha256sum') - - - -def get_required_extensions(extensions): +def get_required_extensions(extensions: List[str]): ext_files = [ - { 'name': 'LDAPAuthentication2', 'file': 'conf/ldap_authentication2.src', }, - { 'name': 'LDAPAuthorization', 'file': 'conf/ldap_authorization.src', }, - # { 'name': 'Auth_remoteuser', 'file': 'conf/ldap_auth_remoteuser.src', }, - { 'name': 'LDAPGroups', 'file': 'conf/ldap_groups.src', }, - { 'name': 'LDAPProvider', 'file': 'conf/ldap_provider.src', }, - { 'name': 'LDAPUserInfo', 'file': 'conf/ldap_userinfo.src', }, - { 'name': 'PluggableAuth', 'file': 'conf/pluggable_auth.src', }, + {'name': 'LDAPAuthentication2', 'file': 'conf/ldap_authentication2.src', }, + {'name': 'LDAPAuthorization', 'file': 'conf/ldap_authorization.src', }, + # {'name': 'Auth_remoteuser', 'file': 'conf/ldap_auth_remoteuser.src', }, + {'name': 'LDAPGroups', 'file': 'conf/ldap_groups.src', }, + {'name': 'LDAPProvider', 'file': 'conf/ldap_provider.src', }, + {'name': 'LDAPUserInfo', 'file': 'conf/ldap_userinfo.src', }, + {'name': 'PluggableAuth', 'file': 'conf/pluggable_auth.src', }, ] for ext in ext_files: file = ext['file'] name = ext['name'] - echo_var = 'source {} ; echo ${}'.format(file, '{}') - current_url = subprocess.check_output(echo_var.format('SOURCE_URL'), shell=True).decode('utf-8').strip() + echo_var = f'source {file} ; echo ${{}}' + current_url = subprocess.check_output( + echo_var.format('SOURCE_URL'), shell=True + ).decode('utf-8').strip() # Search for corresponding in extensions - matching_extension_urls = [ url for url in extensions if name in url ] + matching_extension_urls = [url for url in extensions if name in url] if len(matching_extension_urls) != 1: - print('ERROR: Could not find an upstream link for extension {}'.format(name)) + print(f'ERROR: Could not find an upstream link for extension {name}') continue - new_url = extensions_host_url + matching_extension_urls[0] + new_url = EXTENSIONS_HOST_URL + matching_extension_urls[0] if current_url == new_url: - print('OK: url is up to date for {}'.format(name)) + print(f'OK: url is up to date for {name}') continue - print('Updating source file for {}'.format(name)) - update_source_file(file, new_url, '000') + print(f'Updating source file for {name}') + update_source_file(file, new_url) - -if __name__ == '__main__': +def main(): + """Main function.""" mediawiki_ext_version = get_mediawiki_ext_version() extensions = get_all_extensions() extensions = get_extensions_for_version(extensions, mediawiki_ext_version) get_required_extensions(extensions) + + +if __name__ == '__main__': + main()