YunoHost-Apps/django-for-runners_ynh
1
0
Fork 0
mirror of https://github.com/YunoHost-Apps/django-for-runners_ynh.git synced 2024-09-03 18:26:16 +02:00
Code Issues Activity Actions 2

Merge pull request #72 from YunoHost-Apps/install-python

Browse source 
Install Python with a python script ;)
This commit is contained in:
Jens Diemer 2024-08-29 09:12:12 +02:00 committed by GitHub
commit c2c4aa9b02
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 301 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

294
conf/install_python.py Executable file
View file

@ -0,0 +1,294 @@
#!/usr/bin/env python3
"""
Setup Python Interpreter
~~~~~~~~~~~~~~~~~~~~~~~~
This script downloads, builds and installs a Python interpreter, but:
- only if the required version is not already installed
- only if the required version is not already built
Download Python source code from official Python FTP server.
Download only over verified HTTPS connection.
Verify the download with the GPG signature, if gpg is available.
Has a CLI interface e.g.:
$ python install_python.py --help
Defaults to Python 3.11 and ~/.local/ as prefix.
"""
from __future__ import annotations
import argparse
import hashlib
import logging
import os
import re
import shlex
import shutil
import ssl
import subprocess
import sys
import tempfile
import urllib.request
from pathlib import Path
assert sys.version_info >= (3, 9), f'Python version {sys.version_info} is too old!'
DEFAULT_MAJOR_VERSION = '3.11'
PY_FTP_INDEX_URL = 'https://www.python.org/ftp/python/'
GPG_KEY_IDS = {
# from: https://www.python.org/downloads/
#
# Thomas Wouters (3.12.x and 3.13.x source files and tags) (key id: A821E680E5FA6305):
'3.13': '7169605F62C751356D054A26A821E680E5FA6305',
'3.12': '7169605F62C751356D054A26A821E680E5FA6305',
#
# Pablo Galindo Salgado (3.10.x and 3.11.x source files and tags) (key id: 64E628F8D684696D):
'3.11': 'A035C8C19219BA821ECEA86B64E628F8D684696D',
'3.10': 'A035C8C19219BA821ECEA86B64E628F8D684696D',
}
# https://docs.python.org/3/using/configure.html#cmdoption-prefix
DEFAULT_INSTALL_PREFIX = '/usr/local'
TEMP_PREFIX = 'setup_python_'
logger = logging.getLogger(__name__)
class TemporaryDirectory:
"""tempfile.TemporaryDirectory in Python 3.9 has no "delete", yet."""
def __init__(self, prefix, delete: bool):
self.prefix = prefix
self.delete = delete
def __enter__(self) -> Path:
self.temp_path = Path(tempfile.mkdtemp(prefix=self.prefix))
return self.temp_path
def __exit__(self, exc_type, exc_val, exc_tb):
if self.delete:
shutil.rmtree(self.temp_path, ignore_errors=True)
if exc_type:
return False
def fetch(url: str) -> bytes:
with urllib.request.urlopen(
url=url,
context=ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH),
) as response:
return response.read()
def get_html_page(url) -> str:
logger.debug(f'Getting HTML page from {url}')
html = fetch(url).decode('utf-8')
assert html, 'Failed to get Python FTP index page'
return html
def extract_versions(*, html, major_version) -> list[str]:
pattern = rf'href="({re.escape(major_version)}\.[0-9]+)'
logger.debug(f'Extracting versions with pattern: {pattern}')
versions = re.findall(pattern, html)
versions.sort(reverse=True)
logger.debug(f'Extracted versions: {versions}')
return versions
def get_latest_versions(*, html, major_version) -> str:
latest_versions = extract_versions(html=html, major_version=major_version)[0]
logger.info(f'Latest version of Python {major_version}: {latest_versions}')
return latest_versions
def run(args, **kwargs):
logger.debug(f'Running: {shlex.join(str(arg) for arg in args)} ({kwargs=})')
return subprocess.run(args, **kwargs)
def run_build_step(args, *, step: str, cwd: Path) -> None:
with tempfile.NamedTemporaryFile(prefix=f'{TEMP_PREFIX}_{step}_', suffix='.txt', delete=False) as temp_file:
logger.info(f'Running: {shlex.join(str(arg) for arg in args)}... Output in {temp_file.name}')
try:
subprocess.run(args, stdout=temp_file, stderr=temp_file, check=True, cwd=cwd)
except subprocess.SubprocessError as err:
logger.error(f'Failed to run {step} step: {err}')
run(['tail', temp_file.name])
raise
def get_python_version(python_bin) -> str:
logger.debug(f'Check {python_bin} version')
full_version = run([python_bin, '--version'], capture_output=True, text=True).stdout.split()[1]
logger.info(f'{python_bin} version: {full_version}')
return full_version
def download2temp(*, temp_path: Path, base_url: str, filename: str) -> Path:
url = f'{base_url}/{filename}'
dst_path = temp_path / filename
logger.info(f'Downloading {url} into {dst_path}...')
dst_path.write_bytes(fetch(url))
logger.info(f'Downloaded {filename} is {dst_path.stat().st_size} Bytes')
return dst_path
def verify_download(*, major_version: str, tar_file_path: Path, asc_file_path: Path):
hash_obj = hashlib.sha256(tar_file_path.read_bytes())
logger.info(f'Downloaded sha256: {hash_obj.hexdigest()}')
if gpg_bin := shutil.which('gpg'):
logger.debug(f'Verifying signature with {gpg_bin}...')
assert major_version in GPG_KEY_IDS, f'No GPG key ID for Python {major_version}'
gpg_key_id = GPG_KEY_IDS[major_version]
run([gpg_bin, '--keyserver', 'hkps://keys.openpgp.org', '--recv-keys', gpg_key_id], check=True)
run([gpg_bin, '--verify', asc_file_path, tar_file_path], check=True)
run(['gpgconf', '--kill', 'all'], check=True)
else:
logger.warning('No GPG verification possible! (gpg not found)')
def install_python(
major_version: str,
*,
write_check: bool = True,
delete_temp: bool = True,
) -> Path:
logger.info(f'Installing Python {major_version} interpreter.')
# Check system Python version
for try_version in (major_version, '3'):
filename = f'python{try_version}'
logger.debug(f'Checking {filename}...')
if python3bin := shutil.which(filename):
if get_python_version(python3bin).startswith(major_version):
logger.info('Python version already installed')
return Path(python3bin)
# Get latest full version number of Python from Python FTP:
py_required_version = get_latest_versions(
html=get_html_page(PY_FTP_INDEX_URL),
major_version=major_version,
)
local_bin_path = Path(DEFAULT_INSTALL_PREFIX) / 'bin'
# Check existing built version of Python in /usr/local/bin
local_python_path = local_bin_path / f'python{major_version}'
if local_python_path.exists() and get_python_version(local_python_path) == py_required_version:
logger.info('Local Python is up to date')
return local_python_path
# Before we start building Python, check if we have write permissions:
if write_check and not os.access(local_bin_path, os.W_OK):
raise PermissionError(f'No write permission to {local_bin_path} (Hint: Call with "sudo" ?!)')
# Download, build and Setup Python
with TemporaryDirectory(prefix=TEMP_PREFIX, delete=delete_temp) as temp_path:
base_url = f'https://www.python.org/ftp/python/{py_required_version}'
tar_filename = f'Python-{py_required_version}.tar.xz'
asc_filename = f'{tar_filename}.asc'
asc_file_path = download2temp(
temp_path=temp_path,
base_url=base_url,
filename=asc_filename,
)
tar_file_path = download2temp(
temp_path=temp_path,
base_url=base_url,
filename=tar_filename,
)
verify_download(
major_version=major_version,
tar_file_path=tar_file_path,
asc_file_path=asc_file_path,
)
tar_bin = shutil.which('tar')
logger.debug(f'Extracting {tar_file_path} with ...')
run([tar_bin, 'xf', tar_file_path], check=True, cwd=temp_path)
extracted_dir = temp_path / f'Python-{py_required_version}'
logger.info(f'Building Python {py_required_version} (may take a while)...')
run_build_step(
['./configure', '--enable-optimizations'],
step='configure',
cwd=extracted_dir,
)
run_build_step(
['make', f'-j{os.cpu_count()}'],
step='make',
cwd=extracted_dir,
)
run_build_step(
['make', 'altinstall'],
step='install',
cwd=extracted_dir,
)
logger.info(f'Python {py_required_version} installed to {local_python_path}')
local_python_version = get_python_version(local_python_path)
assert local_python_version == py_required_version, f'{local_python_version} is not {py_required_version}'
return local_python_path
def main() -> Path:
parser = argparse.ArgumentParser(
description='Setup Python Interpreter',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'major_version',
nargs=argparse.OPTIONAL,
default=DEFAULT_MAJOR_VERSION,
choices=sorted(GPG_KEY_IDS.keys()),
help='Specify the Python version',
)
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Increase verbosity level (can be used multiple times, e.g.: -vv)',
)
parser.add_argument(
'--skip-temp-deletion',
action='store_true',
help='Skip deletion of temporary files created during build steps',
)
parser.add_argument(
'--skip-write-check',
action='store_true',
help='Skip the test for write permission to /usr/local/bin',
)
args = parser.parse_args()
verbose2level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
logging.basicConfig(
level=verbose2level.get(args.verbose, logging.DEBUG),
format='%(levelname)9s %(message)s',
stream=sys.stderr,
)
logger.debug(f'Arguments: {args}')
return install_python(
major_version=args.major_version,
write_check=not args.skip_write_check,
delete_temp=not args.skip_temp_deletion,
)
if __name__ == '__main__':
python_path = main()
print(python_path)

84
scripts/_common.sh
View file

@ -40,90 +40,16 @@ log_file="${log_path}/${app}.log"
#==================================================================================
# myynh_install_python() Borrowed from:
# https://github.com/YunoHost-Apps/homeassistant_ynh/blob/master/scripts/_common.sh
# Until we get a newer Python in YunoHost, see:
# https://forum.yunohost.org/t/use-newer-python-than-3-9/22568
#==================================================================================
py_required_major=3.11
py_required_version=$(curl -Ls https://www.python.org/ftp/python/ \
| grep '>'$py_required_major | cut -d '/' -f 2 \
| cut -d '>' -f 2 | sort -rV | head -n 1) #3.11.8
PY_REQUIRED_MAJOR=3.11
myynh_install_python() {
# Declare an array to define the options of this helper.
local legacy_args=u
local -A args_array=( [p]=python= )
local python
# Manage arguments with getopts
ynh_handle_getopts_args "$@"
ynh_print_info --message="Install latest Python v${PY_REQUIRED_MAJOR}..."
# Check python version from APT
local py_apt_version=$(python3 --version | cut -d ' ' -f 2)
# Usefull variables
local python_major=${python%.*}
# Check existing built version of python in /usr/local/bin
if [ -e "/usr/local/bin/python$python_major" ]
then
local py_built_version=$(/usr/local/bin/python$python_major --version \
| cut -d ' ' -f 2)
else
local py_built_version=0
fi
# Compare version
if $(dpkg --compare-versions $py_apt_version ge $python)
then
# APT >= Required
ynh_print_info --message="Using provided python3..."
py_app_version="python3"
else
# Either python already built or to build
if $(dpkg --compare-versions $py_built_version ge $python)
then
# Built >= Required
py_app_version="/usr/local/bin/python${py_built_version%.*}"
ynh_print_info --message="Using already used python3 built version: $py_app_version"
else
# APT < Minimal & Actual < Minimal => Build & install Python into /usr/local/bin
ynh_print_info --message="Building $python (may take a while)..."
# Store current direcotry
local MY_DIR=$(pwd)
# Create a temp direcotry
tmpdir="$(mktemp --directory)"
cd "$tmpdir"
# Download
wget --output-document="Python-$python.tar.xz" \
"https://www.python.org/ftp/python/$python/Python-$python.tar.xz" 2>&1
# Extract
tar xf "Python-$python.tar.xz"
# Install
cd "Python-$python"
./configure --enable-optimizations
ynh_exec_warn_less make -j4
ynh_exec_warn_less make altinstall
# Go back to working directory
cd "$MY_DIR"
# Clean
ynh_secure_remove "$tmpdir"
# Set version
py_app_version="/usr/local/bin/python$python_major"
fi
fi
# Save python version in settings
ynh_app_setting_set --app=$app --key=python --value="$python"
ynh_exec_warn_less python3 "$data_dir/install_python.py" -vv ${PY_REQUIRED_MAJOR}
py_app_version=$(python3 "$data_dir/install_python.py" ${PY_REQUIRED_MAJOR})
# Print some version information:
ynh_print_info --message="Python version: $($py_app_version -VV)"
@ -134,7 +60,7 @@ myynh_install_python() {
myynh_setup_python_venv() {
# Install Python if needed:
myynh_install_python --python="$py_required_version"
myynh_install_python
# Create a virtualenv with python installed by myynh_install_python():
# Skip pip because of: https://github.com/YunoHost/issues/issues/1960

1
scripts/install
View file

@ -68,6 +68,7 @@ ynh_use_logrotate --logfile="$log_file" --specific_user=$app
# PYTHON VIRTUALENV
#=================================================
ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45
cp ../conf/install_python.py "$data_dir/install_python.py"
cp ../conf/requirements.txt "$data_dir/requirements.txt"
myynh_setup_python_venv

1
scripts/upgrade
View file

@ -51,6 +51,7 @@ ynh_add_systemd_config --service=$app --template="systemd.service"
# PYTHON VIRTUALENV
#=================================================
ynh_script_progression --message="Create and setup Python virtualenv..." --weight=45
cp ../conf/install_python.py "$data_dir/install_python.py"
cp ../conf/requirements.txt "$data_dir/requirements.txt"
myynh_setup_python_venv