From 9117c6de27ca4bd0c654edfe8e3e5fb979f6bcf8 Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Sun, 4 Aug 2024 19:31:38 +0200 Subject: [PATCH] Use pyrun for missing Python 3.11 Download and setup missing Python versions using eGenix PyRun from: https://github.com/eGenix/egenix-pyrun/ --- dev-cli.py | 31 +++++---- manifest.toml | 2 +- scripts/_common.sh | 103 ++-------------------------- scripts/setup-pyrun.py | 151 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 110 deletions(-) create mode 100755 scripts/setup-pyrun.py diff --git a/dev-cli.py b/dev-cli.py index 059bf12..05318f9 100755 --- a/dev-cli.py +++ b/dev-cli.py @@ -20,22 +20,27 @@ def print_no_pip_error(): print('Hint: "apt-get install python3-venv"\n') -try: - from ensurepip import version -except ModuleNotFoundError as err: - print(err) - print('-' * 100) - print_no_pip_error() - raise -else: - if not version(): - print_no_pip_error() - sys.exit(-1) - - assert sys.version_info >= (3, 11), f'Python version {sys.version_info} is too old!' +try: + import pip # noqa +except ModuleNotFoundError: + try: + from ensurepip import version + except ModuleNotFoundError as err: + print(err) + print('-' * 100) + print_no_pip_error() + raise + else: + if not version(): + print_no_pip_error() + sys.exit(-1) +else: + print(f'pip version: {pip.__version__}') + + if sys.platform == 'win32': # wtf # Files under Windows, e.g.: .../.venv/Scripts/python.exe BIN_NAME = 'Scripts' diff --git a/manifest.toml b/manifest.toml index 7c12aec..1930e8d 100644 --- a/manifest.toml +++ b/manifest.toml @@ -100,7 +100,7 @@ ram.runtime = "50M" # **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, [resources.apt] # https://yunohost.org/en/packaging_apps_resources#apt # This will automatically install/uninstall the following apt packages - packages = "build-essential, python3-dev, python3-pip, python3-venv, git, libpq-dev, postgresql, postgresql-contrib, redis-server, checkinstall, libssl-dev, openssl" + packages = "python3, git, libpq-dev, postgresql, postgresql-contrib, redis-server, libssl-dev, openssl" [resources.database] # https://yunohost.org/en/packaging_apps_resources#database diff --git a/scripts/_common.sh b/scripts/_common.sh index 5a653db..ab8f6a7 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -38,107 +38,18 @@ log_file="${log_path}/${app}.log" # HELPERS #================================================= - -#================================================================================== -# myynh_install_python() Borrowed from: -# https://github.com/YunoHost-Apps/homeassistant_ynh/blob/master/scripts/_common.sh -# Until we get a newer Python in YunoHost, see: -# https://forum.yunohost.org/t/use-newer-python-than-3-9/22568 -#================================================================================== -py_required_major=3.11 -py_required_version=$(curl -Ls https://www.python.org/ftp/python/ \ - | grep '>'$py_required_major | cut -d '/' -f 2 \ - | cut -d '>' -f 2 | sort -rV | head -n 1) #3.11.8 - -myynh_install_python() { - # Declare an array to define the options of this helper. - local legacy_args=u - local -A args_array=( [p]=python= ) - local python - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - - # Check python version from APT - local py_apt_version=$(python3 --version | cut -d ' ' -f 2) - - # Usefull variables - local python_major=${python%.*} - - # Check existing built version of python in /usr/local/bin - if [ -e "/usr/local/bin/python$python_major" ] - then - local py_built_version=$(/usr/local/bin/python$python_major --version \ - | cut -d ' ' -f 2) - else - local py_built_version=0 - fi - - # Compare version - if $(dpkg --compare-versions $py_apt_version ge $python) - then - # APT >= Required - ynh_print_info --message="Using provided python3..." - - py_app_version="python3" - - else - # Either python already built or to build - if $(dpkg --compare-versions $py_built_version ge $python) - then - # Built >= Required - ynh_print_info --message="Using already used python3 built version..." - - py_app_version="/usr/local/bin/python${py_built_version%.*}" - - else - # APT < Minimal & Actual < Minimal => Build & install Python into /usr/local/bin - ynh_print_info --message="Building python (may take a while)..." - - # Store current direcotry - local MY_DIR=$(pwd) - - # Create a temp direcotry - tmpdir="$(mktemp --directory)" - cd "$tmpdir" - - # Download - wget --output-document="Python-$python.tar.xz" \ - "https://www.python.org/ftp/python/$python/Python-$python.tar.xz" 2>&1 - - # Extract - tar xf "Python-$python.tar.xz" - - # Install - cd "Python-$python" - ./configure --enable-optimizations - ynh_exec_warn_less make -j4 - ynh_exec_warn_less make altinstall - - # Go back to working directory - cd "$MY_DIR" - - # Clean - ynh_secure_remove "$tmpdir" - - # Set version - py_app_version="/usr/local/bin/python$python_major" - fi - fi - # Save python version in settings - ynh_app_setting_set --app=$app --key=python --value="$python" -} -#================================================================================== -#================================================================================== - myynh_setup_python_venv() { # Always recreate everything fresh with current python version ynh_secure_remove "$data_dir/venv" - myynh_install_python --python="$py_required_version" + # Install PyRun: https://github.com/eGenix/egenix-pyrun/ + python3 setup-pyrun.py --version 3.11 --destination $data_dir - # Create a virtualenv with python installed by myynh_install_python(): - # Skip pip because of: https://github.com/YunoHost/issues/issues/1960 - $py_app_version -m venv --without-pip "$data_dir/venv" + # Now PyRun should be installed: + "$data_dir/pyrun3.11/bin/python3.11" -V + + # Create a Python Virtualenv + "$data_dir/pyrun3.11/bin/python3.11" -m virtualenv "$data_dir/venv" chown -c -R "$app:" "$data_dir" diff --git a/scripts/setup-pyrun.py b/scripts/setup-pyrun.py new file mode 100755 index 0000000..7f5c4a1 --- /dev/null +++ b/scripts/setup-pyrun.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +""" + Download and setup missing Python versions using eGenix PyRun from: + + https://github.com/eGenix/egenix-pyrun/ +""" + +import dataclasses +import json +import shutil +import subprocess +import sys +import tarfile +import urllib.request +from argparse import ArgumentParser +from pathlib import Path +from pprint import pprint + +from urllib3.util import parse_url + + +DEFAULT_VERSION = '3.11' + + +GET_PIP_URL = 'https://bootstrap.pypa.io/get-pip.py' # https://github.com/eGenix/egenix-pyrun/issues/11 + + +@dataclasses.dataclass +class ReleaseInfo: + version: str + urls: list[str] + + def get(self, *, version): + for url in self.urls: + if f'-py{version}_' in url: + return url + + +def get_pyrun_release_info() -> ReleaseInfo: + api_url = "https://api.github.com/repos/eGenix/egenix-pyrun/releases" + with urllib.request.urlopen(api_url) as response: + data = response.read().decode() + releases = json.loads(data) + + latest_release = releases[0] + # pprint(latest_release) + + print(latest_release['html_url']) + print(latest_release['tag_name']) + print(latest_release['name']) + + urls = [] + for asset in latest_release['assets']: + urls.append(asset['browser_download_url']) + + return ReleaseInfo(version=latest_release['name'], urls=urls) + + +def setup_pyrun(*, pyrun_version: str, destination: str): + print('_' * 100) + dest_path = Path(destination).expanduser() + dest_path.mkdir(parents=True, exist_ok=True) + print(f'Setup pyrun for Python {pyrun_version} to: {dest_path}') + + release_info = get_pyrun_release_info() + pprint(release_info) + + python_bin_name = f'python{pyrun_version}' + path = shutil.which(python_bin_name) + if path: + path = Path(path).resolve() + print(f'Found {python_bin_name} at {path}') + print(path.name) + if path.name.startswith('pyrun'): + print(f'{python_bin_name} is pyrun -> update') + else: + sys.exit(0) + + url = release_info.get(version=pyrun_version) + if not url: + print(f'No PyRun release found for Python {pyrun_version}') + sys.exit(1) + + print(f'Download {url}') + + subprocess.check_call( + ['wget', '--timestamp', url], + cwd=dest_path, + ) + + filename = Path(parse_url(url).path).name + print(f'{filename=}') + tgz_path = dest_path / filename + assert tgz_path.is_file(), f'{tgz_path=}' + + final_path = dest_path / f'pyrun{pyrun_version}/' + print(f'Extract {tgz_path} to {final_path}...') + final_path.mkdir(parents=False, exist_ok=True) + with tarfile.open(tgz_path, "r:gz") as tgz_file: + for member in tgz_file.getmembers(): + if ".." in member.name or member.name.startswith("/"): + raise ValueError(f"Unsafe file path detected: {member.name}") + tgz_file.extractall(path=final_path) + + tgz_path.unlink() + + pyrun_bin = final_path / 'bin' / f'pyrun{pyrun_version}' + + print(f'Install pip/virtualenv for Python {pyrun_version}') + # FIXME: # https://github.com/eGenix/egenix-pyrun/issues/11 + subprocess.check_call( + ['wget', '--timestamp', GET_PIP_URL], + cwd=dest_path, + ) + get_pip_path = dest_path / 'get-pip.py' + + subprocess.check_call( + [pyrun_bin, get_pip_path], + ) + get_pip_path.unlink() + subprocess.check_call( + [pyrun_bin, '-m', 'pip', 'install', '-U', 'virtualenv'], + ) + print('\n') + + print(f'Check {pyrun_bin}:') + subprocess.check_call([pyrun_bin, '-V']) + + print('\n') + + +if __name__ == '__main__': + parser = ArgumentParser(description='Setup PyRun for a specific Python version.') + parser.add_argument( + '--version', + type=str, + default=DEFAULT_VERSION, + help=f'Python version to setup with PyRun (default: {DEFAULT_VERSION}),', + ) + parser.add_argument( + '--destination', + type=str, + help=f'Destination path to store PyRun (e.g.: ~/.local/)', + ) + args = parser.parse_args() + + setup_pyrun( + pyrun_version=args.version, + destination=args.destination, + )