diff --git a/conf/install_python.py b/conf/install_python.py new file mode 100755 index 0000000..dc72e83 --- /dev/null +++ b/conf/install_python.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +""" + Setup Python Interpreter + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This script downloads, builds and installs a Python interpreter, but: + - only if the required version is not already installed + - only if the required version is not already built + + Download Python source code from official Python FTP server. + Download only over verified HTTPS connection. + Verify the download with the GPG signature, if gpg is available. + + Has a CLI interface e.g.: + + $ python install_python.py --help + + Defaults to Python 3.11 and ~/.local/ as prefix. +""" +from __future__ import annotations + +import argparse +import hashlib +import logging +import os +import re +import shlex +import shutil +import ssl +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + + +assert sys.version_info >= (3, 9), f'Python version {sys.version_info} is too old!' + + +DEFAULT_MAJOR_VERSION = '3.11' +PY_FTP_INDEX_URL = 'https://www.python.org/ftp/python/' + + +GPG_KEY_IDS = { + # from: https://www.python.org/downloads/ + # + # Thomas Wouters (3.12.x and 3.13.x source files and tags) (key id: A821E680E5FA6305): + '3.13': '7169605F62C751356D054A26A821E680E5FA6305', + '3.12': '7169605F62C751356D054A26A821E680E5FA6305', + # + # Pablo Galindo Salgado (3.10.x and 3.11.x source files and tags) (key id: 64E628F8D684696D): + '3.11': 'A035C8C19219BA821ECEA86B64E628F8D684696D', + '3.10': 'A035C8C19219BA821ECEA86B64E628F8D684696D', +} + +# https://docs.python.org/3/using/configure.html#cmdoption-prefix +DEFAULT_INSTALL_PREFIX = '/usr/local' + +TEMP_PREFIX = 'setup_python_' + +logger = logging.getLogger(__name__) + + +class TemporaryDirectory: + """tempfile.TemporaryDirectory in Python 3.9 has no "delete", yet.""" + + def __init__(self, prefix, delete: bool): + self.prefix = prefix + self.delete = delete + + def __enter__(self) -> Path: + self.temp_path = Path(tempfile.mkdtemp(prefix=self.prefix)) + return self.temp_path + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.delete: + shutil.rmtree(self.temp_path, ignore_errors=True) + if exc_type: + return False + + +def fetch(url: str) -> bytes: + with urllib.request.urlopen( + url=url, + context=ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH), + ) as response: + return response.read() + + +def get_html_page(url) -> str: + logger.debug(f'Getting HTML page from {url}') + html = fetch(url).decode('utf-8') + assert html, 'Failed to get Python FTP index page' + return html + + +def extract_versions(*, html, major_version) -> list[str]: + pattern = rf'href="({re.escape(major_version)}\.[0-9]+)' + logger.debug(f'Extracting versions with pattern: {pattern}') + versions = re.findall(pattern, html) + versions.sort(reverse=True) + logger.debug(f'Extracted versions: {versions}') + return versions + + +def get_latest_versions(*, html, major_version) -> str: + latest_versions = extract_versions(html=html, major_version=major_version)[0] + logger.info(f'Latest version of Python {major_version}: {latest_versions}') + return latest_versions + + +def run(args, **kwargs): + logger.debug(f'Running: {shlex.join(str(arg) for arg in args)} ({kwargs=})') + return subprocess.run(args, **kwargs) + + +def run_build_step(args, *, step: str, cwd: Path) -> None: + with tempfile.NamedTemporaryFile(prefix=f'{TEMP_PREFIX}_{step}_', suffix='.txt', delete=False) as temp_file: + logger.info(f'Running: {shlex.join(str(arg) for arg in args)}... Output in {temp_file.name}') + try: + subprocess.run(args, stdout=temp_file, stderr=temp_file, check=True, cwd=cwd) + except subprocess.SubprocessError as err: + logger.error(f'Failed to run {step} step: {err}') + run(['tail', temp_file.name]) + raise + + +def get_python_version(python_bin) -> str: + logger.debug(f'Check {python_bin} version') + full_version = run([python_bin, '--version'], capture_output=True, text=True).stdout.split()[1] + logger.info(f'{python_bin} version: {full_version}') + return full_version + + +def download2temp(*, temp_path: Path, base_url: str, filename: str) -> Path: + url = f'{base_url}/{filename}' + dst_path = temp_path / filename + logger.info(f'Downloading {url} into {dst_path}...') + dst_path.write_bytes(fetch(url)) + logger.info(f'Downloaded {filename} is {dst_path.stat().st_size} Bytes') + return dst_path + + +def verify_download(*, major_version: str, tar_file_path: Path, asc_file_path: Path): + hash_obj = hashlib.sha256(tar_file_path.read_bytes()) + logger.info(f'Downloaded sha256: {hash_obj.hexdigest()}') + + if gpg_bin := shutil.which('gpg'): + logger.debug(f'Verifying signature with {gpg_bin}...') + assert major_version in GPG_KEY_IDS, f'No GPG key ID for Python {major_version}' + gpg_key_id = GPG_KEY_IDS[major_version] + run([gpg_bin, '--keyserver', 'hkps://keys.openpgp.org', '--recv-keys', gpg_key_id], check=True) + run([gpg_bin, '--verify', asc_file_path, tar_file_path], check=True) + run(['gpgconf', '--kill', 'all'], check=True) + else: + logger.warning('No GPG verification possible! (gpg not found)') + + +def install_python( + major_version: str, + *, + write_check: bool = True, + delete_temp: bool = True, +) -> Path: + logger.info(f'Installing Python {major_version} interpreter.') + + # Check system Python version + for try_version in (major_version, '3'): + filename = f'python{try_version}' + logger.debug(f'Checking {filename}...') + if python3bin := shutil.which(filename): + if get_python_version(python3bin).startswith(major_version): + logger.info('Python version already installed') + return Path(python3bin) + + # Get latest full version number of Python from Python FTP: + py_required_version = get_latest_versions( + html=get_html_page(PY_FTP_INDEX_URL), + major_version=major_version, + ) + + local_bin_path = Path(DEFAULT_INSTALL_PREFIX) / 'bin' + + # Check existing built version of Python in /usr/local/bin + local_python_path = local_bin_path / f'python{major_version}' + if local_python_path.exists() and get_python_version(local_python_path) == py_required_version: + logger.info('Local Python is up to date') + return local_python_path + + # Before we start building Python, check if we have write permissions: + if write_check and not os.access(local_bin_path, os.W_OK): + raise PermissionError(f'No write permission to {local_bin_path} (Hint: Call with "sudo" ?!)') + + # Download, build and Setup Python + with TemporaryDirectory(prefix=TEMP_PREFIX, delete=delete_temp) as temp_path: + base_url = f'https://www.python.org/ftp/python/{py_required_version}' + + tar_filename = f'Python-{py_required_version}.tar.xz' + asc_filename = f'{tar_filename}.asc' + asc_file_path = download2temp( + temp_path=temp_path, + base_url=base_url, + filename=asc_filename, + ) + tar_file_path = download2temp( + temp_path=temp_path, + base_url=base_url, + filename=tar_filename, + ) + verify_download( + major_version=major_version, + tar_file_path=tar_file_path, + asc_file_path=asc_file_path, + ) + + tar_bin = shutil.which('tar') + logger.debug(f'Extracting {tar_file_path} with ...') + run([tar_bin, 'xf', tar_file_path], check=True, cwd=temp_path) + extracted_dir = temp_path / f'Python-{py_required_version}' + + logger.info(f'Building Python {py_required_version} (may take a while)...') + + run_build_step( + ['./configure', '--enable-optimizations'], + step='configure', + cwd=extracted_dir, + ) + run_build_step( + ['make', f'-j{os.cpu_count()}'], + step='make', + cwd=extracted_dir, + ) + run_build_step( + ['make', 'altinstall'], + step='install', + cwd=extracted_dir, + ) + + logger.info(f'Python {py_required_version} installed to {local_python_path}') + + local_python_version = get_python_version(local_python_path) + assert local_python_version == py_required_version, f'{local_python_version} is not {py_required_version}' + + return local_python_path + + +def main() -> Path: + parser = argparse.ArgumentParser( + description='Setup Python Interpreter', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + 'major_version', + nargs=argparse.OPTIONAL, + default=DEFAULT_MAJOR_VERSION, + choices=sorted(GPG_KEY_IDS.keys()), + help='Specify the Python version', + ) + parser.add_argument( + '-v', + '--verbose', + action='count', + default=0, + help='Increase verbosity level (can be used multiple times, e.g.: -vv)', + ) + parser.add_argument( + '--skip-temp-deletion', + action='store_true', + help='Skip deletion of temporary files created during build steps', + ) + parser.add_argument( + '--skip-write-check', + action='store_true', + help='Skip the test for write permission to /usr/local/bin', + ) + args = parser.parse_args() + verbose2level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + logging.basicConfig( + level=verbose2level.get(args.verbose, logging.DEBUG), + format='%(levelname)9s %(message)s', + stream=sys.stderr, + ) + logger.debug(f'Arguments: {args}') + return install_python( + major_version=args.major_version, + write_check=not args.skip_write_check, + delete_temp=not args.skip_temp_deletion, + ) + + +if __name__ == '__main__': + python_path = main() + print(python_path) diff --git a/scripts/_common.sh b/scripts/_common.sh index 958cf6c..a4ac505 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -40,90 +40,16 @@ log_file="${log_path}/${app}.log" #================================================================================== -# myynh_install_python() Borrowed from: -# https://github.com/YunoHost-Apps/homeassistant_ynh/blob/master/scripts/_common.sh # Until we get a newer Python in YunoHost, see: # https://forum.yunohost.org/t/use-newer-python-than-3-9/22568 #================================================================================== -py_required_major=3.11 -py_required_version=$(curl -Ls https://www.python.org/ftp/python/ \ - | grep '>'$py_required_major | cut -d '/' -f 2 \ - | cut -d '>' -f 2 | sort -rV | head -n 1) #3.11.8 +PY_REQUIRED_MAJOR=3.11 myynh_install_python() { - # Declare an array to define the options of this helper. - local legacy_args=u - local -A args_array=( [p]=python= ) - local python - # Manage arguments with getopts - ynh_handle_getopts_args "$@" + ynh_print_info --message="Install latest Python v${PY_REQUIRED_MAJOR}..." - # Check python version from APT - local py_apt_version=$(python3 --version | cut -d ' ' -f 2) - - # Usefull variables - local python_major=${python%.*} - - # Check existing built version of python in /usr/local/bin - if [ -e "/usr/local/bin/python$python_major" ] - then - local py_built_version=$(/usr/local/bin/python$python_major --version \ - | cut -d ' ' -f 2) - else - local py_built_version=0 - fi - - # Compare version - if $(dpkg --compare-versions $py_apt_version ge $python) - then - # APT >= Required - ynh_print_info --message="Using provided python3..." - - py_app_version="python3" - - else - # Either python already built or to build - if $(dpkg --compare-versions $py_built_version ge $python) - then - # Built >= Required - py_app_version="/usr/local/bin/python${py_built_version%.*}" - ynh_print_info --message="Using already used python3 built version: $py_app_version" - else - # APT < Minimal & Actual < Minimal => Build & install Python into /usr/local/bin - ynh_print_info --message="Building $python (may take a while)..." - - # Store current direcotry - local MY_DIR=$(pwd) - - # Create a temp direcotry - tmpdir="$(mktemp --directory)" - cd "$tmpdir" - - # Download - wget --output-document="Python-$python.tar.xz" \ - "https://www.python.org/ftp/python/$python/Python-$python.tar.xz" 2>&1 - - # Extract - tar xf "Python-$python.tar.xz" - - # Install - cd "Python-$python" - ./configure --enable-optimizations - ynh_exec_warn_less make -j4 - ynh_exec_warn_less make altinstall - - # Go back to working directory - cd "$MY_DIR" - - # Clean - ynh_secure_remove "$tmpdir" - - # Set version - py_app_version="/usr/local/bin/python$python_major" - fi - fi - # Save python version in settings - ynh_app_setting_set --app=$app --key=python --value="$python" + ynh_exec_warn_less python3 "$data_dir/install_python.py" -vv ${PY_REQUIRED_MAJOR} + py_app_version=$(python3 "$data_dir/install_python.py" ${PY_REQUIRED_MAJOR}) # Print some version information: ynh_print_info --message="Python version: $($py_app_version -VV)" @@ -134,7 +60,7 @@ myynh_install_python() { myynh_setup_python_venv() { # Install Python if needed: - myynh_install_python --python="$py_required_version" + myynh_install_python # Create a virtualenv with python installed by myynh_install_python(): # Skip pip because of: https://github.com/YunoHost/issues/issues/1960 diff --git a/scripts/install b/scripts/install index 0d26ed4..4a59ffe 100755 --- a/scripts/install +++ b/scripts/install @@ -68,6 +68,7 @@ ynh_use_logrotate --logfile="$log_file" --specific_user=$app # PYTHON VIRTUALENV #================================================= ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45 +cp ../conf/install_python.py "$data_dir/install_python.py" cp ../conf/requirements.txt "$data_dir/requirements.txt" myynh_setup_python_venv diff --git a/scripts/upgrade b/scripts/upgrade index 848a09b..5a8c75f 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -51,6 +51,7 @@ ynh_add_systemd_config --service=$app --template="systemd.service" # PYTHON VIRTUALENV #================================================= ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45 +cp ../conf/install_python.py "$data_dir/install_python.py" cp ../conf/requirements.txt "$data_dir/requirements.txt" myynh_setup_python_venv