mediawiki_ynh/update_extensions.py

#!/usr/bin/env python3

import json
from os import replace
import urllib.request
from html.parser import HTMLParser
import subprocess
import hashlib
import sys, fileinput

extensions_host_url = 'https://extdist.wmflabs.org/dist/extensions/'

def get_all_extensions():
    webpage = urllib.request.urlopen(extensions_host_url).read().decode('utf-8')

    class MyHTMLParser(HTMLParser):
        links = []
        def handle_starttag(self, tag, attrs):
            # Only parse the 'anchor' tag.
            if tag == 'a':
                # Check the list of defined attributes.
                for name, value in attrs:
                    # If href is defined, print it.
                    if name == "href":
                        self.links.append(value)

    parser = MyHTMLParser()
    parser.feed(webpage)
    return parser.links

def get_mediawiki_ext_version():
    with open('manifest.json') as manifest_json:
        manifest = json.load(manifest_json)
    mediawiki_version = manifest['version'].split('~')[0]
    mediawiki_ext_version = '_'.join(mediawiki_version.split('.')[0:2])
    return mediawiki_ext_version


def get_extensions_for_version(extensions, version):
    exts = [ ext for ext in extensions if version in ext ]
    return exts



###############################################################################

def sha256sum(filename):
    sha256_hash = hashlib.sha256()
    with open(filename,"rb") as f:
        # Read and update hash string value in blocks of 4K
        for byte_block in iter(lambda: f.read(4096),b""):
            sha256_hash.update(byte_block)
    return sha256_hash.hexdigest()


def replace_line_startingwith(file, startingwith, new_line):
    for line in fileinput.input(file, inplace=1):
        if line.startswith(startingwith):
            line = new_line + '\n'
        sys.stdout.write(line)
    pass

def update_source_file(srcfile, url, shasum):
    filename = url.rsplit('/', 1)[1]
    urllib.request.urlretrieve(url, filename)
    hash = sha256sum(filename)

    replace_line_startingwith(srcfile, 'SOURCE_URL=', 'SOURCE_URL={}'.format(url))
    replace_line_startingwith(srcfile, 'SOURCE_SUM=', 'SOURCE_SUM={}'.format(hash))
    replace_line_startingwith(srcfile, 'SOURCE_SUM_PRG=', 'SOURCE_SUM_PRG=sha256sum')





def get_required_extensions(extensions):
    ext_files = [
        { 'name': 'LDAPAuthentication2',    'file': 'conf/ldap_authentication2.src', },
        { 'name': 'LDAPAuthorization',      'file': 'conf/ldap_authorization.src', },
        { 'name': 'Auth_remoteuser',        'file': 'conf/ldap_auth_remoteuser.src', },
        { 'name': 'LDAPGroups',             'file': 'conf/ldap_groups.src', },
        { 'name': 'LDAPProvider',           'file': 'conf/ldap_provider.src', },
        { 'name': 'LDAPUserInfo',           'file': 'conf/ldap_userinfo.src', },
        { 'name': 'PluggableAuth',          'file': 'conf/pluggable_auth.src', },
    ]

    for ext in ext_files:
        file = ext['file']
        name = ext['name']

        echo_var = 'source {} ; echo ${}'.format(file, '{}')
        current_url = subprocess.check_output(echo_var.format('SOURCE_URL'), shell=True).decode('utf-8').strip()

        # Search for corresponding in extensions
        matching_extension_urls = [ url for url in extensions if name in url ]

        if len(matching_extension_urls) != 1:
            print('ERROR: Could not find an upstream link for extension {}'.format(name))
            continue

        new_url = extensions_host_url + matching_extension_urls[0]

        if current_url == new_url:
            print('OK: url is up to date for {}'.format(name))
            continue

        print('Updating source file for {}'.format(name))
        update_source_file(file, new_url, '000')



if __name__ == '__main__':
    mediawiki_ext_version = get_mediawiki_ext_version()
    extensions = get_all_extensions()
    extensions = get_extensions_for_version(extensions, mediawiki_ext_version)
    get_required_extensions(extensions)