diff --git a/conf/app.src b/conf/app.src index 7a7baef..7172b3b 100644 --- a/conf/app.src +++ b/conf/app.src @@ -1,5 +1,5 @@ -SOURCE_URL=https://releases.wikimedia.org/mediawiki/1.37/mediawiki-1.37.0.tar.gz -SOURCE_SUM=dd1ad227c5bbbc833d851c01727ee8706a39e4aa31e0f49ce45e50257ef9b97f +SOURCE_URL=https://releases.wikimedia.org/mediawiki/1.37/mediawiki-1.37.1.tar.gz +SOURCE_SUM=53436e92dc30adb14b6797d8139d20696a5461525f38a375c83e435cf042e2e7 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=true diff --git a/conf/ldap_authentication2.src b/conf/ldap_authentication2.src index 38ac5fa..5697f84 100644 --- a/conf/ldap_authentication2.src +++ b/conf/ldap_authentication2.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthentication2-REL1_36-8fce73c.tar.gz -SOURCE_SUM=238ed3dd414798d28c762cf0bff003ff2de1b0be50644864f3ea2b9bffd77440 +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthentication2-REL1_35-dabdf22.tar.gz +SOURCE_SUM=e53b1bd791b288e0531bdec5ca58386b7d7287899f4677073bf5c93a2b778b3f SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/ldap_authorization.src b/conf/ldap_authorization.src index 15557ab..0417ad8 100644 --- a/conf/ldap_authorization.src +++ b/conf/ldap_authorization.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthorization-REL1_36-bc6b46e.tar.gz -SOURCE_SUM=eadd43bfefc0aa7d00794beff11736224e794b27e03160d17d35863d2fe56d5c +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthorization-REL1_37-ba9cdb5.tar.gz +SOURCE_SUM=b5134407376f33e0c43c7c85fb965757a6ffd83d13280ff26e84d1ced4747bb8 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/ldap_groups.src b/conf/ldap_groups.src index 17d703f..cfc4b8e 100644 --- a/conf/ldap_groups.src +++ b/conf/ldap_groups.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPGroups-REL1_36-14d3b1a.tar.gz -SOURCE_SUM=c8f9d2546117430f1360029d81e6a11d28a15553d7e823e00654c3ebd09be465 +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPGroups-REL1_37-f622040.tar.gz +SOURCE_SUM=e198ac995b383d461086c848fb6adf1064b593647fd61199daa22af56b60df47 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/ldap_provider.src b/conf/ldap_provider.src index c48f17c..954a2fd 100644 --- a/conf/ldap_provider.src +++ b/conf/ldap_provider.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPProvider-REL1_36-15aa175.tar.gz -SOURCE_SUM=214d40665c78c4193ef39ae78875af2090a16a5ad6420dfd2d25fe9e3929f9f3 +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPProvider-REL1_35-a801161.tar.gz +SOURCE_SUM=e0ef72d68ba6c89123212bdd06a3739b4f733d3b45e5caa3a0a295a8225a15ea SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/ldap_userinfo.src b/conf/ldap_userinfo.src index 15fbd32..69d7157 100644 --- a/conf/ldap_userinfo.src +++ b/conf/ldap_userinfo.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPUserInfo-REL1_36-e8a8823.tar.gz -SOURCE_SUM=450dfd47e1da297eace5d76aeba1b7820023d210209633a3434e043a69f40798 +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPUserInfo-REL1_37-a4b1bcf.tar.gz +SOURCE_SUM=0e236e5d0498079c722e1bcff398b7e46ef5a165ebe0f201ebbc6195d066746e SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/pluggable_auth.src b/conf/pluggable_auth.src index f8bd863..daac1a4 100644 --- a/conf/pluggable_auth.src +++ b/conf/pluggable_auth.src @@ -1,5 +1,6 @@ -SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_36-70ed593.tar.gz -SOURCE_SUM=40fcc36ba02195dab2bcdd7ed90f86d6dab9520225678b8d8408510fda753334 +SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_37-5757eca.tar.gz +SOURCE_SUM=d81edf9033db3a98bc51384b43f6926dfda78b456844db31d800f6d5884511e7 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false +SOURCE_EXTRACT=true diff --git a/conf/update_sources.py b/conf/update_sources.py new file mode 100755 index 0000000..7ffe12f --- /dev/null +++ b/conf/update_sources.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import List, Optional +import hashlib +import json +import urllib +from html.parser import HTMLParser +from packaging import version +import requests + +# GENERIC CODE + +# Don't edit this file manually, but {program} instead. +SOURCE_TEMPLATE = """SOURCE_URL={url} +SOURCE_SUM={sha256sum} +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=tar.gz +SOURCE_IN_SUBDIR={source_in_subdir} +SOURCE_EXTRACT={source_extract} +""" + +def generate_source(url: str, output_name: str, source_extract=True, source_in_subdir=True) -> None: + with NamedTemporaryFile() as tempfile: + response = requests.get(url) + response.raise_for_status() + with open(tempfile.name, "wb") as datafile: + for chunk in response.iter_content(chunk_size=1024): + datafile.write(chunk) + + sha256_hash = hashlib.sha256() + with open(tempfile.name, "rb") as datafile: + # Read and update hash string value in blocks of 4K + for byte_block in iter(lambda: datafile.read(4096),b""): + sha256_hash.update(byte_block) + sha256sum = sha256_hash.hexdigest() + + with open(Path(__file__).parent / output_name, "w", encoding="utf-8") as output: + output.write(SOURCE_TEMPLATE.format( + program=Path(__file__).name, url=url, sha256sum=sha256sum, + source_in_subdir=("true" if source_in_subdir else "false"), + source_extract=("true" if source_extract else "false") + )) + +# SPECIFIC TO MEDIAWIKI + +VERSION = "1.37.1" +EXTENSION_VERSION = "_".join(VERSION.split(".")[0:2]) +EXTENSIONS_HOST_URL = "https://extdist.wmflabs.org/dist/extensions/" + +EXTENSIONS = { + "ldap_authentication2": "LDAPAuthentication2", + "ldap_authorization": "LDAPAuthorization", + # "ldap_auth_remoteuser": "Auth_remoteuser", + "ldap_groups": "LDAPGroups", + "ldap_provider": "LDAPProvider", + "ldap_userinfo": "LDAPUserInfo", + "pluggable_auth": "PluggableAuth", +} + + +def get_all_extensions() -> List[str]: + """Get all available extensions.""" + with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page: + webpage = page.read().decode("utf-8") + + class MyHTMLParser(HTMLParser): + links = [] + def handle_starttag(self, tag, attrs): + if tag == "a": + for name, value in attrs: + if name == "href": + self.links.append(value) + + parser = MyHTMLParser() + parser.feed(webpage) + return parser.links + +def find_valid_ext(all_exts: List[str], name: str, max_ver: str) -> Optional[str]: + def version_of(ext): + return version.parse(ext.split("-")[1].replace("_", ".").replace("REL", "")) + + found_exts = [ext for ext in all_exts if ext.startswith(name)] + return max(found_exts, key=version_of) if found_exts else None + +def main(): + print(f'Updating source file for Mediawiki...') + version_dir = ".".join(VERSION.split(".")[0:2]) + generate_source( + f"https://releases.wikimedia.org/mediawiki/{version_dir}/mediawiki-{VERSION}.tar.gz", + "app.src" + ) + + all_extensions = get_all_extensions() + for file, name in EXTENSIONS.items(): + print(f'Updating source file for {name}') + ext = find_valid_ext(all_extensions, name, VERSION) + if ext is None: + print(f'ERROR: Could not find an upstream link for extension {name}') + else: + new_url = EXTENSIONS_HOST_URL + ext + generate_source(new_url, file + ".src", source_in_subdir=False) + + + +if __name__ == "__main__": + main() diff --git a/update_extensions.py b/update_extensions.py deleted file mode 100755 index 2f0bb52..0000000 --- a/update_extensions.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -""" -This tool updates conf/*.src according to the available extension version at wmflabs.org -""" - -import sys -import json -import urllib.request -from html.parser import HTMLParser -import subprocess -import hashlib -import fileinput -from typing import List - -EXTENSIONS_HOST_URL = 'https://extdist.wmflabs.org/dist/extensions/' - - -def get_all_extensions() -> List[str]: - """Get all available extensions.""" - with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page: - webpage = page.read().decode('utf-8') - - class MyHTMLParser(HTMLParser): - """Custom HTMLParser""" - links = [] - - def handle_starttag(self, tag, attrs): - # Only parse the 'anchor' tag. - if tag == 'a': - # Check the list of defined attributes. - for name, value in attrs: - # If href is defined, print it. - if name == "href": - self.links.append(value) - - parser = MyHTMLParser() - parser.feed(webpage) - return parser.links - - -def get_mediawiki_ext_version() -> str: - """Returns the mediawiki version for extensions.""" - with open('manifest.json', encoding='utf-8') as manifest_json: - manifest = json.load(manifest_json) - mediawiki_version = manifest['version'].split('~')[0] - mediawiki_ext_version = '_'.join(mediawiki_version.split('.')[0:2]) - return mediawiki_ext_version - - -def get_extensions_for_version(extensions: List[str], version: str) -> List[str]: - """Returns available extensions compatible with mediawiki version.""" - exts = [ext for ext in extensions if version in ext] - return exts - - -############################################################################### - -def sha256sum(filename: str) -> str: - """Calculate the sha256 of a file.""" - sha256_hash = hashlib.sha256() - with open(filename, "rb") as file: - # Read and update hash string value in blocks of 4K - for byte_block in iter(lambda: file.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - - -def replace_line_startingwith(file: str, startingwith: str, new_line: str): - """""" - for line in fileinput.input(file, inplace=1): - if line.startswith(startingwith): - line = new_line + '\n' - sys.stdout.write(line) - - -def update_source_file(srcfile: str, url: str): - filename = url.rsplit('/', 1)[1] - urllib.request.urlretrieve(url, filename) - hashsum = sha256sum(filename) - - replace_line_startingwith(srcfile, 'SOURCE_URL=', f'SOURCE_URL={url}') - replace_line_startingwith(srcfile, 'SOURCE_SUM=', f'SOURCE_SUM={hashsum}') - replace_line_startingwith(srcfile, 'SOURCE_SUM_PRG=', 'SOURCE_SUM_PRG=sha256sum') - - -def get_required_extensions(extensions: List[str]): - ext_files = [ - {'name': 'LDAPAuthentication2', 'file': 'conf/ldap_authentication2.src', }, - {'name': 'LDAPAuthorization', 'file': 'conf/ldap_authorization.src', }, - # {'name': 'Auth_remoteuser', 'file': 'conf/ldap_auth_remoteuser.src', }, - {'name': 'LDAPGroups', 'file': 'conf/ldap_groups.src', }, - {'name': 'LDAPProvider', 'file': 'conf/ldap_provider.src', }, - {'name': 'LDAPUserInfo', 'file': 'conf/ldap_userinfo.src', }, - {'name': 'PluggableAuth', 'file': 'conf/pluggable_auth.src', }, - ] - - for ext in ext_files: - file = ext['file'] - name = ext['name'] - - echo_var = f'source {file} ; echo ${{}}' - current_url = subprocess.check_output( - echo_var.format('SOURCE_URL'), shell=True - ).decode('utf-8').strip() - - # Search for corresponding in extensions - matching_extension_urls = [url for url in extensions if name in url] - - if len(matching_extension_urls) != 1: - print(f'ERROR: Could not find an upstream link for extension {name}') - continue - - new_url = EXTENSIONS_HOST_URL + matching_extension_urls[0] - - if current_url == new_url: - print(f'OK: url is up to date for {name}') - continue - - print(f'Updating source file for {name}') - update_source_file(file, new_url) - - -def main(): - """Main function.""" - mediawiki_ext_version = get_mediawiki_ext_version() - extensions = get_all_extensions() - extensions = get_extensions_for_version(extensions, mediawiki_ext_version) - get_required_extensions(extensions) - - -if __name__ == '__main__': - main()