Merge pull request #72 from YunoHost-Apps/install-python

Install Python with a python script ;)

conf/install_python.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+
+"""
+    Setup Python Interpreter
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This script downloads, builds and installs a Python interpreter, but:
+    - only if the required version is not already installed
+    - only if the required version is not already built
+
+    Download Python source code from official Python FTP server.
+    Download only over verified HTTPS connection.
+    Verify the download with the GPG signature, if gpg is available.
+
+    Has a CLI interface e.g.:
+
+        $ python install_python.py --help
+
+    Defaults to Python 3.11 and ~/.local/ as prefix.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import logging
+import os
+import re
+import shlex
+import shutil
+import ssl
+import subprocess
+import sys
+import tempfile
+import urllib.request
+from pathlib import Path
+
+
+assert sys.version_info >= (3, 9), f'Python version {sys.version_info} is too old!'
+
+
+DEFAULT_MAJOR_VERSION = '3.11'
+PY_FTP_INDEX_URL = 'https://www.python.org/ftp/python/'
+
+
+GPG_KEY_IDS = {
+    # from: https://www.python.org/downloads/
+    #
+    # Thomas Wouters (3.12.x and 3.13.x source files and tags) (key id: A821E680E5FA6305):
+    '3.13': '7169605F62C751356D054A26A821E680E5FA6305',
+    '3.12': '7169605F62C751356D054A26A821E680E5FA6305',
+    #
+    # Pablo Galindo Salgado (3.10.x and 3.11.x source files and tags) (key id: 64E628F8D684696D):
+    '3.11': 'A035C8C19219BA821ECEA86B64E628F8D684696D',
+    '3.10': 'A035C8C19219BA821ECEA86B64E628F8D684696D',
+}
+
+# https://docs.python.org/3/using/configure.html#cmdoption-prefix
+DEFAULT_INSTALL_PREFIX = '/usr/local'
+
+TEMP_PREFIX = 'setup_python_'
+
+logger = logging.getLogger(__name__)
+
+
+class TemporaryDirectory:
+    """tempfile.TemporaryDirectory in Python 3.9 has no "delete", yet."""
+
+    def __init__(self, prefix, delete: bool):
+        self.prefix = prefix
+        self.delete = delete
+
+    def __enter__(self) -> Path:
+        self.temp_path = Path(tempfile.mkdtemp(prefix=self.prefix))
+        return self.temp_path
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.delete:
+            shutil.rmtree(self.temp_path, ignore_errors=True)
+        if exc_type:
+            return False
+
+
+def fetch(url: str) -> bytes:
+    with urllib.request.urlopen(
+        url=url,
+        context=ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH),
+    ) as response:
+        return response.read()
+
+
+def get_html_page(url) -> str:
+    logger.debug(f'Getting HTML page from {url}')
+    html = fetch(url).decode('utf-8')
+    assert html, 'Failed to get Python FTP index page'
+    return html
+
+
+def extract_versions(*, html, major_version) -> list[str]:
+    pattern = rf'href="({re.escape(major_version)}\.[0-9]+)'
+    logger.debug(f'Extracting versions with pattern: {pattern}')
+    versions = re.findall(pattern, html)
+    versions.sort(reverse=True)
+    logger.debug(f'Extracted versions: {versions}')
+    return versions
+
+
+def get_latest_versions(*, html, major_version) -> str:
+    latest_versions = extract_versions(html=html, major_version=major_version)[0]
+    logger.info(f'Latest version of Python {major_version}: {latest_versions}')
+    return latest_versions
+
+
+def run(args, **kwargs):
+    logger.debug(f'Running: {shlex.join(str(arg) for arg in args)} ({kwargs=})')
+    return subprocess.run(args, **kwargs)
+
+
+def run_build_step(args, *, step: str, cwd: Path) -> None:
+    with tempfile.NamedTemporaryFile(prefix=f'{TEMP_PREFIX}_{step}_', suffix='.txt', delete=False) as temp_file:
+        logger.info(f'Running: {shlex.join(str(arg) for arg in args)}... Output in {temp_file.name}')
+        try:
+            subprocess.run(args, stdout=temp_file, stderr=temp_file, check=True, cwd=cwd)
+        except subprocess.SubprocessError as err:
+            logger.error(f'Failed to run {step} step: {err}')
+            run(['tail', temp_file.name])
+            raise
+
+
+def get_python_version(python_bin) -> str:
+    logger.debug(f'Check {python_bin} version')
+    full_version = run([python_bin, '--version'], capture_output=True, text=True).stdout.split()[1]
+    logger.info(f'{python_bin} version: {full_version}')
+    return full_version
+
+
+def download2temp(*, temp_path: Path, base_url: str, filename: str) -> Path:
+    url = f'{base_url}/{filename}'
+    dst_path = temp_path / filename
+    logger.info(f'Downloading {url} into {dst_path}...')
+    dst_path.write_bytes(fetch(url))
+    logger.info(f'Downloaded {filename} is {dst_path.stat().st_size} Bytes')
+    return dst_path
+
+
+def verify_download(*, major_version: str, tar_file_path: Path, asc_file_path: Path):
+    hash_obj = hashlib.sha256(tar_file_path.read_bytes())
+    logger.info(f'Downloaded sha256: {hash_obj.hexdigest()}')
+
+    if gpg_bin := shutil.which('gpg'):
+        logger.debug(f'Verifying signature with {gpg_bin}...')
+        assert major_version in GPG_KEY_IDS, f'No GPG key ID for Python {major_version}'
+        gpg_key_id = GPG_KEY_IDS[major_version]
+        run([gpg_bin, '--keyserver', 'hkps://keys.openpgp.org', '--recv-keys', gpg_key_id], check=True)
+        run([gpg_bin, '--verify', asc_file_path, tar_file_path], check=True)
+        run(['gpgconf', '--kill', 'all'], check=True)
+    else:
+        logger.warning('No GPG verification possible! (gpg not found)')
+
+
+def install_python(
+    major_version: str,
+    *,
+    write_check: bool = True,
+    delete_temp: bool = True,
+) -> Path:
+    logger.info(f'Installing Python {major_version} interpreter.')
+
+    # Check system Python version
+    for try_version in (major_version, '3'):
+        filename = f'python{try_version}'
+        logger.debug(f'Checking {filename}...')
+        if python3bin := shutil.which(filename):
+            if get_python_version(python3bin).startswith(major_version):
+                logger.info('Python version already installed')
+                return Path(python3bin)
+
+    # Get latest full version number of Python from Python FTP:
+    py_required_version = get_latest_versions(
+        html=get_html_page(PY_FTP_INDEX_URL),
+        major_version=major_version,
+    )
+
+    local_bin_path = Path(DEFAULT_INSTALL_PREFIX) / 'bin'
+
+    # Check existing built version of Python in /usr/local/bin
+    local_python_path = local_bin_path / f'python{major_version}'
+    if local_python_path.exists() and get_python_version(local_python_path) == py_required_version:
+        logger.info('Local Python is up to date')
+        return local_python_path
+
+    # Before we start building Python, check if we have write permissions:
+    if write_check and not os.access(local_bin_path, os.W_OK):
+        raise PermissionError(f'No write permission to {local_bin_path} (Hint: Call with "sudo" ?!)')
+
+    # Download, build and Setup Python
+    with TemporaryDirectory(prefix=TEMP_PREFIX, delete=delete_temp) as temp_path:
+        base_url = f'https://www.python.org/ftp/python/{py_required_version}'
+
+        tar_filename = f'Python-{py_required_version}.tar.xz'
+        asc_filename = f'{tar_filename}.asc'
+        asc_file_path = download2temp(
+            temp_path=temp_path,
+            base_url=base_url,
+            filename=asc_filename,
+        )
+        tar_file_path = download2temp(
+            temp_path=temp_path,
+            base_url=base_url,
+            filename=tar_filename,
+        )
+        verify_download(
+            major_version=major_version,
+            tar_file_path=tar_file_path,
+            asc_file_path=asc_file_path,
+        )
+
+        tar_bin = shutil.which('tar')
+        logger.debug(f'Extracting {tar_file_path} with ...')
+        run([tar_bin, 'xf', tar_file_path], check=True, cwd=temp_path)
+        extracted_dir = temp_path / f'Python-{py_required_version}'
+
+        logger.info(f'Building Python {py_required_version} (may take a while)...')
+
+        run_build_step(
+            ['./configure', '--enable-optimizations'],
+            step='configure',
+            cwd=extracted_dir,
+        )
+        run_build_step(
+            ['make', f'-j{os.cpu_count()}'],
+            step='make',
+            cwd=extracted_dir,
+        )
+        run_build_step(
+            ['make', 'altinstall'],
+            step='install',
+            cwd=extracted_dir,
+        )
+
+    logger.info(f'Python {py_required_version} installed to {local_python_path}')
+
+    local_python_version = get_python_version(local_python_path)
+    assert local_python_version == py_required_version, f'{local_python_version} is not {py_required_version}'
+
+    return local_python_path
+
+
+def main() -> Path:
+    parser = argparse.ArgumentParser(
+        description='Setup Python Interpreter',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        'major_version',
+        nargs=argparse.OPTIONAL,
+        default=DEFAULT_MAJOR_VERSION,
+        choices=sorted(GPG_KEY_IDS.keys()),
+        help='Specify the Python version',
+    )
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='count',
+        default=0,
+        help='Increase verbosity level (can be used multiple times, e.g.: -vv)',
+    )
+    parser.add_argument(
+        '--skip-temp-deletion',
+        action='store_true',
+        help='Skip deletion of temporary files created during build steps',
+    )
+    parser.add_argument(
+        '--skip-write-check',
+        action='store_true',
+        help='Skip the test for write permission to /usr/local/bin',
+    )
+    args = parser.parse_args()
+    verbose2level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
+    logging.basicConfig(
+        level=verbose2level.get(args.verbose, logging.DEBUG),
+        format='%(levelname)9s %(message)s',
+        stream=sys.stderr,
+    )
+    logger.debug(f'Arguments: {args}')
+    return install_python(
+        major_version=args.major_version,
+        write_check=not args.skip_write_check,
+        delete_temp=not args.skip_temp_deletion,
+    )
+
+
+if __name__ == '__main__':
+    python_path = main()
+    print(python_path)

scripts/_common.sh

@@ -40,90 +40,16 @@ log_file="${log_path}/${app}.log"
 
 
 #==================================================================================
-# myynh_install_python() Borrowed from:
-# https://github.com/YunoHost-Apps/homeassistant_ynh/blob/master/scripts/_common.sh
 # Until we get a newer Python in YunoHost, see:
 # https://forum.yunohost.org/t/use-newer-python-than-3-9/22568
 #==================================================================================
-py_required_major=3.11
+PY_REQUIRED_MAJOR=3.11
-py_required_version=$(curl -Ls https://www.python.org/ftp/python/ \
-						| grep '>'$py_required_major  | cut -d '/' -f 2 \
-						| cut -d '>' -f 2 | sort -rV | head -n 1) #3.11.8
 
 myynh_install_python() {
-	# Declare an array to define the options of this helper.
+    ynh_print_info --message="Install latest Python v${PY_REQUIRED_MAJOR}..."
-	local legacy_args=u
-	local -A args_array=( [p]=python= )
-	local python
-	# Manage arguments with getopts
-	ynh_handle_getopts_args "$@"
 
-	# Check python version from APT
+    ynh_exec_warn_less python3 "$data_dir/install_python.py" -vv ${PY_REQUIRED_MAJOR}
-	local py_apt_version=$(python3 --version | cut -d ' ' -f 2)
+	py_app_version=$(python3 "$data_dir/install_python.py" ${PY_REQUIRED_MAJOR})
-
-	# Usefull variables
-	local python_major=${python%.*}
-
-	# Check existing built version of python in /usr/local/bin
-	if [ -e "/usr/local/bin/python$python_major" ]
-	then
-		local py_built_version=$(/usr/local/bin/python$python_major --version \
-			| cut -d ' ' -f 2)
-	else
-		local py_built_version=0
-	fi
-
-	# Compare version
-	if $(dpkg --compare-versions $py_apt_version ge $python)
-	then
-		# APT >= Required
-		ynh_print_info --message="Using provided python3..."
-
-		py_app_version="python3"
-
-	else
-		# Either python already built or to build
-		if $(dpkg --compare-versions $py_built_version ge $python)
-		then
-			# Built >= Required
-			py_app_version="/usr/local/bin/python${py_built_version%.*}"
-			ynh_print_info --message="Using already used python3 built version: $py_app_version"
-		else
-			# APT < Minimal & Actual < Minimal => Build & install Python into /usr/local/bin
-			ynh_print_info --message="Building $python (may take a while)..."
-
-			# Store current direcotry
-			local MY_DIR=$(pwd)
-
-			# Create a temp direcotry
-			tmpdir="$(mktemp --directory)"
-			cd "$tmpdir"
-
-			# Download
-			wget --output-document="Python-$python.tar.xz" \
-				"https://www.python.org/ftp/python/$python/Python-$python.tar.xz" 2>&1
-
-			# Extract
-			tar xf "Python-$python.tar.xz"
-
-			# Install
-			cd "Python-$python"
-			./configure --enable-optimizations
-			ynh_exec_warn_less make -j4
-			ynh_exec_warn_less make altinstall
-
-			# Go back to working directory
-			cd "$MY_DIR"
-
-			# Clean
-			ynh_secure_remove "$tmpdir"
-
-			# Set version
-			py_app_version="/usr/local/bin/python$python_major"
-		fi
-	fi
-	# Save python version in settings
-	ynh_app_setting_set --app=$app --key=python --value="$python"
 
 	# Print some version information:
 	ynh_print_info --message="Python version: $($py_app_version -VV)"
@@ -134,7 +60,7 @@ myynh_install_python() {
 
 myynh_setup_python_venv() {
     # Install Python if needed:
-    myynh_install_python --python="$py_required_version"
+    myynh_install_python
 
     # Create a virtualenv with python installed by myynh_install_python():
     # Skip pip because of: https://github.com/YunoHost/issues/issues/1960

scripts/install

@@ -68,6 +68,7 @@ ynh_use_logrotate --logfile="$log_file" --specific_user=$app
 # PYTHON VIRTUALENV
 #=================================================
 ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45
+cp ../conf/install_python.py "$data_dir/install_python.py"
 cp ../conf/requirements.txt "$data_dir/requirements.txt"
 myynh_setup_python_venv
 

scripts/upgrade

@@ -51,6 +51,7 @@ ynh_add_systemd_config --service=$app --template="systemd.service"
 # PYTHON VIRTUALENV
 #=================================================
 ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45
+cp ../conf/install_python.py "$data_dir/install_python.py"
 cp ../conf/requirements.txt "$data_dir/requirements.txt"
 myynh_setup_python_venv