Update source urls, app bump to 1.37.1

conf/app.src

@@ -1,5 +1,5 @@
-SOURCE_URL=https://releases.wikimedia.org/mediawiki/1.37/mediawiki-1.37.0.tar.gz
+SOURCE_URL=https://releases.wikimedia.org/mediawiki/1.37/mediawiki-1.37.1.tar.gz
-SOURCE_SUM=dd1ad227c5bbbc833d851c01727ee8706a39e4aa31e0f49ce45e50257ef9b97f
+SOURCE_SUM=53436e92dc30adb14b6797d8139d20696a5461525f38a375c83e435cf042e2e7
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=true

conf/ldap_authentication2.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthentication2-REL1_36-8fce73c.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthentication2-REL1_35-dabdf22.tar.gz
-SOURCE_SUM=238ed3dd414798d28c762cf0bff003ff2de1b0be50644864f3ea2b9bffd77440
+SOURCE_SUM=e53b1bd791b288e0531bdec5ca58386b7d7287899f4677073bf5c93a2b778b3f
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/ldap_authorization.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthorization-REL1_36-bc6b46e.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPAuthorization-REL1_37-ba9cdb5.tar.gz
-SOURCE_SUM=eadd43bfefc0aa7d00794beff11736224e794b27e03160d17d35863d2fe56d5c
+SOURCE_SUM=b5134407376f33e0c43c7c85fb965757a6ffd83d13280ff26e84d1ced4747bb8
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/ldap_groups.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPGroups-REL1_36-14d3b1a.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPGroups-REL1_37-f622040.tar.gz
-SOURCE_SUM=c8f9d2546117430f1360029d81e6a11d28a15553d7e823e00654c3ebd09be465
+SOURCE_SUM=e198ac995b383d461086c848fb6adf1064b593647fd61199daa22af56b60df47
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/ldap_provider.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPProvider-REL1_36-15aa175.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPProvider-REL1_35-a801161.tar.gz
-SOURCE_SUM=214d40665c78c4193ef39ae78875af2090a16a5ad6420dfd2d25fe9e3929f9f3
+SOURCE_SUM=e0ef72d68ba6c89123212bdd06a3739b4f733d3b45e5caa3a0a295a8225a15ea
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/ldap_userinfo.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPUserInfo-REL1_36-e8a8823.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/LDAPUserInfo-REL1_37-a4b1bcf.tar.gz
-SOURCE_SUM=450dfd47e1da297eace5d76aeba1b7820023d210209633a3434e043a69f40798
+SOURCE_SUM=0e236e5d0498079c722e1bcff398b7e46ef5a165ebe0f201ebbc6195d066746e
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/pluggable_auth.src

@@ -1,5 +1,6 @@
-SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_36-70ed593.tar.gz
+SOURCE_URL=https://extdist.wmflabs.org/dist/extensions/PluggableAuth-REL1_37-5757eca.tar.gz
-SOURCE_SUM=40fcc36ba02195dab2bcdd7ed90f86d6dab9520225678b8d8408510fda753334
+SOURCE_SUM=d81edf9033db3a98bc51384b43f6926dfda78b456844db31d800f6d5884511e7
 SOURCE_SUM_PRG=sha256sum
 SOURCE_FORMAT=tar.gz
 SOURCE_IN_SUBDIR=false
+SOURCE_EXTRACT=true

conf/update_sources.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import List, Optional
+import hashlib
+import json
+import urllib
+from html.parser import HTMLParser
+from packaging import version
+import requests
+
+# GENERIC CODE
+
+# Don't edit this file manually, but {program} instead.
+SOURCE_TEMPLATE = """SOURCE_URL={url}
+SOURCE_SUM={sha256sum}
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=tar.gz
+SOURCE_IN_SUBDIR={source_in_subdir}
+SOURCE_EXTRACT={source_extract}
+"""
+
+def generate_source(url: str, output_name: str, source_extract=True, source_in_subdir=True) -> None:
+    with NamedTemporaryFile() as tempfile:
+        response = requests.get(url)
+        response.raise_for_status()
+        with open(tempfile.name, "wb") as datafile:
+            for chunk in response.iter_content(chunk_size=1024):
+                datafile.write(chunk)
+
+        sha256_hash = hashlib.sha256()
+        with open(tempfile.name, "rb") as datafile:
+            # Read and update hash string value in blocks of 4K
+            for byte_block in iter(lambda: datafile.read(4096),b""):
+                sha256_hash.update(byte_block)
+        sha256sum = sha256_hash.hexdigest()
+
+    with open(Path(__file__).parent / output_name, "w", encoding="utf-8") as output:
+        output.write(SOURCE_TEMPLATE.format(
+            program=Path(__file__).name, url=url, sha256sum=sha256sum,
+            source_in_subdir=("true" if source_in_subdir else "false"),
+            source_extract=("true" if source_extract else "false")
+        ))
+
+# SPECIFIC TO MEDIAWIKI
+
+VERSION = "1.37.1"
+EXTENSION_VERSION = "_".join(VERSION.split(".")[0:2])
+EXTENSIONS_HOST_URL = "https://extdist.wmflabs.org/dist/extensions/"
+
+EXTENSIONS = {
+    "ldap_authentication2": "LDAPAuthentication2",
+    "ldap_authorization": "LDAPAuthorization",
+    # "ldap_auth_remoteuser": "Auth_remoteuser",
+    "ldap_groups": "LDAPGroups",
+    "ldap_provider": "LDAPProvider",
+    "ldap_userinfo": "LDAPUserInfo",
+    "pluggable_auth": "PluggableAuth",
+}
+
+
+def get_all_extensions() -> List[str]:
+    """Get all available extensions."""
+    with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page:
+        webpage = page.read().decode("utf-8")
+
+    class MyHTMLParser(HTMLParser):
+        links = []
+        def handle_starttag(self, tag, attrs):
+            if tag == "a":
+                for name, value in attrs:
+                    if name == "href":
+                        self.links.append(value)
+
+    parser = MyHTMLParser()
+    parser.feed(webpage)
+    return parser.links
+
+def find_valid_ext(all_exts: List[str], name: str, max_ver: str) -> Optional[str]:
+    def version_of(ext):
+        return version.parse(ext.split("-")[1].replace("_", ".").replace("REL", ""))
+
+    found_exts = [ext for ext in all_exts if ext.startswith(name)]
+    return max(found_exts, key=version_of) if found_exts else None
+
+def main():
+    print(f'Updating source file for Mediawiki...')
+    version_dir = ".".join(VERSION.split(".")[0:2])
+    generate_source(
+        f"https://releases.wikimedia.org/mediawiki/{version_dir}/mediawiki-{VERSION}.tar.gz",
+        "app.src"
+    )
+
+    all_extensions = get_all_extensions()
+    for file, name in EXTENSIONS.items():
+        print(f'Updating source file for {name}')
+        ext = find_valid_ext(all_extensions, name, VERSION)
+        if ext is None:
+            print(f'ERROR: Could not find an upstream link for extension {name}')
+        else:
+            new_url = EXTENSIONS_HOST_URL + ext
+            generate_source(new_url, file + ".src", source_in_subdir=False)
+
+
+
+if __name__ == "__main__":
+    main()

update_extensions.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-"""
-This tool updates conf/*.src according to the available extension version at wmflabs.org
-"""
-
-import sys
-import json
-import urllib.request
-from html.parser import HTMLParser
-import subprocess
-import hashlib
-import fileinput
-from typing import List
-
-EXTENSIONS_HOST_URL = 'https://extdist.wmflabs.org/dist/extensions/'
-
-
-def get_all_extensions() -> List[str]:
-    """Get all available extensions."""
-    with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page:
-        webpage = page.read().decode('utf-8')
-
-    class MyHTMLParser(HTMLParser):
-        """Custom HTMLParser"""
-        links = []
-
-        def handle_starttag(self, tag, attrs):
-            # Only parse the 'anchor' tag.
-            if tag == 'a':
-                # Check the list of defined attributes.
-                for name, value in attrs:
-                    # If href is defined, print it.
-                    if name == "href":
-                        self.links.append(value)
-
-    parser = MyHTMLParser()
-    parser.feed(webpage)
-    return parser.links
-
-
-def get_mediawiki_ext_version() -> str:
-    """Returns the mediawiki version for extensions."""
-    with open('manifest.json', encoding='utf-8') as manifest_json:
-        manifest = json.load(manifest_json)
-    mediawiki_version = manifest['version'].split('~')[0]
-    mediawiki_ext_version = '_'.join(mediawiki_version.split('.')[0:2])
-    return mediawiki_ext_version
-
-
-def get_extensions_for_version(extensions: List[str], version: str) -> List[str]:
-    """Returns available extensions compatible with mediawiki version."""
-    exts = [ext for ext in extensions if version in ext]
-    return exts
-
-
-###############################################################################
-
-def sha256sum(filename: str) -> str:
-    """Calculate the sha256 of a file."""
-    sha256_hash = hashlib.sha256()
-    with open(filename, "rb") as file:
-        # Read and update hash string value in blocks of 4K
-        for byte_block in iter(lambda: file.read(4096), b""):
-            sha256_hash.update(byte_block)
-    return sha256_hash.hexdigest()
-
-
-def replace_line_startingwith(file: str, startingwith: str, new_line: str):
-    """"""
-    for line in fileinput.input(file, inplace=1):
-        if line.startswith(startingwith):
-            line = new_line + '\n'
-        sys.stdout.write(line)
-
-
-def update_source_file(srcfile: str, url: str):
-    filename = url.rsplit('/', 1)[1]
-    urllib.request.urlretrieve(url, filename)
-    hashsum = sha256sum(filename)
-
-    replace_line_startingwith(srcfile, 'SOURCE_URL=', f'SOURCE_URL={url}')
-    replace_line_startingwith(srcfile, 'SOURCE_SUM=', f'SOURCE_SUM={hashsum}')
-    replace_line_startingwith(srcfile, 'SOURCE_SUM_PRG=', 'SOURCE_SUM_PRG=sha256sum')
-
-
-def get_required_extensions(extensions: List[str]):
-    ext_files = [
-        {'name': 'LDAPAuthentication2', 'file': 'conf/ldap_authentication2.src', },
-        {'name': 'LDAPAuthorization',   'file': 'conf/ldap_authorization.src', },
-        # {'name': 'Auth_remoteuser',     'file': 'conf/ldap_auth_remoteuser.src', },
-        {'name': 'LDAPGroups',          'file': 'conf/ldap_groups.src', },
-        {'name': 'LDAPProvider',        'file': 'conf/ldap_provider.src', },
-        {'name': 'LDAPUserInfo',        'file': 'conf/ldap_userinfo.src', },
-        {'name': 'PluggableAuth',       'file': 'conf/pluggable_auth.src', },
-    ]
-
-    for ext in ext_files:
-        file = ext['file']
-        name = ext['name']
-
-        echo_var = f'source {file} ; echo ${{}}'
-        current_url = subprocess.check_output(
-            echo_var.format('SOURCE_URL'), shell=True
-        ).decode('utf-8').strip()
-
-        # Search for corresponding in extensions
-        matching_extension_urls = [url for url in extensions if name in url]
-
-        if len(matching_extension_urls) != 1:
-            print(f'ERROR: Could not find an upstream link for extension {name}')
-            continue
-
-        new_url = EXTENSIONS_HOST_URL + matching_extension_urls[0]
-
-        if current_url == new_url:
-            print(f'OK: url is up to date for {name}')
-            continue
-
-        print(f'Updating source file for {name}')
-        update_source_file(file, new_url)
-
-
-def main():
-    """Main function."""
-    mediawiki_ext_version = get_mediawiki_ext_version()
-    extensions = get_all_extensions()
-    extensions = get_extensions_for_version(extensions, mediawiki_ext_version)
-    get_required_extensions(extensions)
-
-
-if __name__ == '__main__':
-    main()