Add script to easily update extensions

update_extensions.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import json
+from os import replace
+import urllib.request
+from html.parser import HTMLParser
+import subprocess
+import hashlib
+import sys, fileinput
+
+extensions_host_url = 'https://extdist.wmflabs.org/dist/extensions/'
+
+def get_all_extensions():
+    webpage = urllib.request.urlopen(extensions_host_url).read().decode('utf-8')
+
+    class MyHTMLParser(HTMLParser):
+        links = []
+        def handle_starttag(self, tag, attrs):
+            # Only parse the 'anchor' tag.
+            if tag == 'a':
+                # Check the list of defined attributes.
+                for name, value in attrs:
+                    # If href is defined, print it.
+                    if name == "href":
+                        self.links.append(value)
+
+    parser = MyHTMLParser()
+    parser.feed(webpage)
+    return parser.links
+
+def get_mediawiki_ext_version():
+    with open('manifest.json') as manifest_json:
+        manifest = json.load(manifest_json)
+    mediawiki_version = manifest['version'].split('~')[0]
+    mediawiki_ext_version = '_'.join(mediawiki_version.split('.')[0:2])
+    return mediawiki_ext_version
+
+
+def get_extensions_for_version(extensions, version):
+    exts = [ ext for ext in extensions if version in ext ]
+    return exts
+
+
+
+###############################################################################
+
+def sha256sum(filename):
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        # Read and update hash string value in blocks of 4K
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+def replace_line_startingwith(file, startingwith, new_line):
+    for line in fileinput.input(file, inplace=1):
+        if line.startswith(startingwith):
+            line = new_line + '\n'
+        sys.stdout.write(line)
+    pass
+
+def update_source_file(srcfile, url, shasum):
+    filename = url.rsplit('/', 1)[1]
+    urllib.request.urlretrieve(url, filename)
+    hash = sha256sum(filename)
+
+    replace_line_startingwith(srcfile, 'SOURCE_URL=', 'SOURCE_URL={}'.format(url))
+    replace_line_startingwith(srcfile, 'SOURCE_SUM=', 'SOURCE_SUM={}'.format(hash))
+    replace_line_startingwith(srcfile, 'SOURCE_SUM_PRG=', 'SOURCE_SUM_PRG=sha256sum')
+
+
+
+
+
+def get_required_extensions(extensions):
+    ext_files = [
+        { 'name': 'LDAPAuthentication2',    'file': 'conf/ldap_authentication2.src', },
+        { 'name': 'LDAPAuthorization',      'file': 'conf/ldap_authorization.src', },
+        { 'name': 'Auth_remoteuser',        'file': 'conf/ldap_auth_remoteuser.src', },
+        { 'name': 'LDAPGroups',             'file': 'conf/ldap_groups.src', },
+        { 'name': 'LDAPProvider',           'file': 'conf/ldap_provider.src', },
+        { 'name': 'LDAPUserInfo',           'file': 'conf/ldap_userinfo.src', },
+        { 'name': 'PluggableAuth',          'file': 'conf/pluggable_auth.src', },
+    ]
+
+    for ext in ext_files:
+        file = ext['file']
+        name = ext['name']
+
+        echo_var = 'source {} ; echo ${}'.format(file, '{}')
+        current_url = subprocess.check_output(echo_var.format('SOURCE_URL'), shell=True).decode('utf-8').strip()
+
+        # Search for corresponding in extensions
+        matching_extension_urls = [ url for url in extensions if name in url ]
+
+        if len(matching_extension_urls) != 1:
+            print('ERROR: Could not find an upstream link for extension {}'.format(name))
+            continue
+
+        new_url = extensions_host_url + matching_extension_urls[0]
+
+        if current_url == new_url:
+            print('OK: url is up to date for {}'.format(name))
+            continue
+
+        print('Updating source file for {}'.format(name))
+        update_source_file(file, new_url, '000')
+
+
+
+if __name__ == '__main__':
+    mediawiki_ext_version = get_mediawiki_ext_version()
+    extensions = get_all_extensions()
+    extensions = get_extensions_for_version(extensions, mediawiki_ext_version)
+    get_required_extensions(extensions)