mediawiki_ynh/conf/update_sources.py

#!/usr/bin/env python3

from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import List, Optional
import hashlib
import json
import urllib
from html.parser import HTMLParser
from packaging import version
import requests

# GENERIC CODE

# Don't edit this file manually, but {program} instead.
SOURCE_TEMPLATE = """SOURCE_URL={url}
SOURCE_SUM={sha256sum}
SOURCE_SUM_PRG=sha256sum
SOURCE_FORMAT=tar.gz
SOURCE_IN_SUBDIR={source_in_subdir}
SOURCE_EXTRACT={source_extract}
"""

def generate_source(url: str, output_name: str, source_extract=True, source_in_subdir=True) -> None:
    with NamedTemporaryFile() as tempfile:
        response = requests.get(url)
        response.raise_for_status()
        with open(tempfile.name, "wb") as datafile:
            for chunk in response.iter_content(chunk_size=1024):
                datafile.write(chunk)

        sha256_hash = hashlib.sha256()
        with open(tempfile.name, "rb") as datafile:
            # Read and update hash string value in blocks of 4K
            for byte_block in iter(lambda: datafile.read(4096),b""):
                sha256_hash.update(byte_block)
        sha256sum = sha256_hash.hexdigest()

    with open(Path(__file__).parent / output_name, "w", encoding="utf-8") as output:
        output.write(SOURCE_TEMPLATE.format(
            program=Path(__file__).name, url=url, sha256sum=sha256sum,
            source_in_subdir=("true" if source_in_subdir else "false"),
            source_extract=("true" if source_extract else "false")
        ))

# SPECIFIC TO MEDIAWIKI

VERSION = "1.37.1"
EXTENSION_VERSION = "_".join(VERSION.split(".")[0:2])
EXTENSIONS_HOST_URL = "https://extdist.wmflabs.org/dist/extensions/"

EXTENSIONS = {
    "ldap_authentication2": "LDAPAuthentication2",
    "ldap_authorization": "LDAPAuthorization",
    # "ldap_auth_remoteuser": "Auth_remoteuser",
    "ldap_groups": "LDAPGroups",
    "ldap_provider": "LDAPProvider",
    "ldap_userinfo": "LDAPUserInfo",
    "pluggable_auth": "PluggableAuth",
}


def get_all_extensions() -> List[str]:
    """Get all available extensions."""
    with urllib.request.urlopen(EXTENSIONS_HOST_URL) as page:
        webpage = page.read().decode("utf-8")

    class MyHTMLParser(HTMLParser):
        links = []
        def handle_starttag(self, tag, attrs):
            if tag == "a":
                for name, value in attrs:
                    if name == "href":
                        self.links.append(value)

    parser = MyHTMLParser()
    parser.feed(webpage)
    return parser.links

def find_valid_ext(all_exts: List[str], name: str, max_ver: str) -> Optional[str]:
    def version_of(ext):
        return version.parse(ext.split("-")[1].replace("_", ".").replace("REL", ""))

    found_exts = [ext for ext in all_exts if ext.startswith(name)]
    return max(found_exts, key=version_of) if found_exts else None

def main():
    print(f'Updating source file for Mediawiki...')
    version_dir = ".".join(VERSION.split(".")[0:2])
    generate_source(
        f"https://releases.wikimedia.org/mediawiki/{version_dir}/mediawiki-{VERSION}.tar.gz",
        "app.src"
    )

    all_extensions = get_all_extensions()
    for file, name in EXTENSIONS.items():
        print(f'Updating source file for {name}')
        ext = find_valid_ext(all_extensions, name, VERSION)
        if ext is None:
            print(f'ERROR: Could not find an upstream link for extension {name}')
        else:
            new_url = EXTENSIONS_HOST_URL + ext
            generate_source(new_url, file + ".src", source_in_subdir=False)



if __name__ == "__main__":
    main()