#!/usr/bin/env python3 """ Download extensions for the current mediawiki version, and update the conf files. """ from typing import List, Optional import hashlib import urllib from html.parser import HTMLParser import tomlkit from packaging import version import requests EXTENSIONS_HOST_URL = "https://extdist.wmflabs.org/dist/extensions/" GITHUB_API_URL = "https://api.github.com/repos" def sha256sum_of_url(url: str) -> str: """Compute checksum without saving the file""" checksum = hashlib.sha256() for chunk in requests.get(url, stream=True, timeout=10).iter_content(): checksum.update(chunk) return checksum.hexdigest() def find_valid_version(all_versions: List[str], max_version: version.Version) -> Optional[str]: """Find the valid extensions for the current mediawiki version""" def version_of(ext): try: return version.parse(ext.replace("_", ".").replace("REL", "")) except version.InvalidVersion: # print(f"Invalid version (this might be normal): {ext}") return version.parse("0.0") def compatible(ext_version: str) -> bool: return version_of(ext_version) <= max_version compatible_versions = filter(compatible, all_versions) if compatible_versions: return max(compatible_versions, key=version_of) if "master" in all_versions: return "master" return None def get_repo(url: str) -> str: return "/".join(url.split("://")[1].split("/")[1:3]) def get_branches(repo: str) -> List[str]: branches = requests.get(f"{GITHUB_API_URL}/{repo}/branches", timeout=10).json() names = [branch["name"] for branch in branches] return names def get_last_commit_of(repo: str, branch: str) -> str: commit = requests.get(f"{GITHUB_API_URL}/{repo}/commits/{branch}", timeout=10).json() return commit["sha"] def main(): print('Updating extensions source files...') with open("manifest.toml", "r", encoding="utf-8") as file: manifest = tomlkit.loads(file.read()) mediawiki_version = version.Version(manifest["version"].value.split("~")[0]) for name, descr in manifest["resources"]["sources"].items(): if "extension" not in descr["url"]: # not an extension continue print(f'Updating source file for {name}') repo = get_repo(descr["url"]) branches = get_branches(repo) branch = find_valid_version(branches, mediawiki_version) if not branch: print("Could not find any valid branch") continue commit = get_last_commit_of(repo, branch) url = f"https://github.com/{repo}/archive/{commit}.tar.gz" manifest["resources"]["sources"][name]["url"] = url manifest["resources"]["sources"][name]["sha256"] = sha256sum_of_url(url) with open("manifest.toml", "w", encoding="utf-8") as manifest_file: manifest_file.write(tomlkit.dumps(manifest)) if __name__ == "__main__": main()