diff --git a/tools/appslib/get_apps_repo.py b/tools/appslib/get_apps_repo.py new file mode 100644 index 00000000..2058fce6 --- /dev/null +++ b/tools/appslib/get_apps_repo.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import os +import argparse +import tempfile +import logging +from pathlib import Path +from typing import Optional +from git import Repo + + +DEFAULT_GIT_REPO = "git@github.com:YunoHost/apps" + + +class TemporaryPath(Path): + """ Just a helper to return agnostically a Path or a TemporaryDirectory """ + def __init__(self, *args, **kwargs): + self.temporary_directory = tempfile.TemporaryDirectory(*args, **kwargs) + Path.__init__(self, self.temporary_directory.name) + + def with_segments(self, *pathsegments): + """ We need to overload this method because it calls type(self) + but we don't want to create multiple TemporaryPaths. + """ + return Path(*pathsegments) + + +APPS_REPO_DIR: Optional[TemporaryPath] = None + +def add_args(parser: argparse.ArgumentParser, required: bool = False, allow_temp: bool = True) -> None: + env_apps_dir_str = os.environ.get("YNH_APPS_DIR") + env_apps_dir = Path(env_apps_dir_str) if env_apps_dir_str is not None else None + + repo_group = parser.add_mutually_exclusive_group(required=required) + if allow_temp: + repo_group.add_argument("-c", "--apps-repo", type=str, default=DEFAULT_GIT_REPO, + help="Git url to clone the 'apps' repository") + repo_group.add_argument("-d", "--apps-dir", type=Path, help="Path to an existing 'apps' repository", default=env_apps_dir) + + +def from_args(args: Optional[argparse.Namespace]) -> Path: + global APPS_REPO_DIR + if APPS_REPO_DIR is not None: + return APPS_REPO_DIR + + assert args is not None + if args.apps_dir is not None: + APPS_REPO_DIR = args.apps_dir + assert APPS_REPO_DIR is not None + return APPS_REPO_DIR + + if args.apps_repo is not None: + tmpdir = TemporaryPath(prefix="yunohost_apps_") + logging.info("Cloning the 'apps' repository...") + repo = Repo.clone_from(args.apps_repo, to_path=tmpdir) + assert repo.working_tree_dir is not None + APPS_REPO_DIR = tmpdir + return APPS_REPO_DIR + + raise RuntimeError("You need to pass either --apps-repo or --apps-dir!") diff --git a/tools/appslib/utils.py b/tools/appslib/utils.py index 7353ce60..c2470c22 100644 --- a/tools/appslib/utils.py +++ b/tools/appslib/utils.py @@ -13,11 +13,6 @@ import toml REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir) -@cache -def apps_repo_root() -> Path: - return Path(__file__).parent.parent.parent - - def git(cmd: list[str], cwd: Optional[Path] = None) -> str: full_cmd = ["git"] if cwd: @@ -41,9 +36,11 @@ def git_repo_age(path: Path) -> Union[bool, int]: @cache -def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]: +def get_catalog(apps_repo: Path | None = None, working_only: bool = False) -> dict[str, dict[str, Any]]: """Load the app catalog and filter out the non-working ones""" - catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) + apps_repo = apps_repo or REPO_APPS_ROOT + + catalog = toml.load((apps_repo / "apps.toml").open("r", encoding="utf-8")) if working_only: catalog = { app: infos diff --git a/tools/autopatches/autopatch.py b/tools/autopatches/autopatch.py index 20190e38..c4627385 100755 --- a/tools/autopatches/autopatch.py +++ b/tools/autopatches/autopatch.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import json import os import subprocess @@ -14,19 +15,20 @@ import toml sys.path.insert(0, str(Path(__file__).parent.parent)) from appslib.utils import ( # noqa: E402 pylint: disable=import-error,wrong-import-position - REPO_APPS_ROOT, get_catalog, ) +TOOLS_DIR = Path(__file__).resolve().parent.parent + my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" os.makedirs(".apps_cache", exist_ok=True) login = ( - (REPO_APPS_ROOT / "tools/.github_login").open("r", encoding="utf-8").read().strip() + (TOOLS_DIR / ".github_login").open("r", encoding="utf-8").read().strip() ) token = ( - (REPO_APPS_ROOT / "tools/.github_token").open("r", encoding="utf-8").read().strip() + (TOOLS_DIR / ".github_token").open("r", encoding="utf-8").read().strip() ) github_api = "https://api.github.com" @@ -193,36 +195,31 @@ def create_pull_request(repo, patch, base_branch, s): def main(): - action = sys.argv[1] - if action == "--help": - print( - """ - Example usage: + parser = argparse.ArgumentParser() + parser.add_argument("the_patch", type=str, nargs="?", help="The name of the patch to apply") + parser.add_argument("--cache", "-b", action="store_true", help="Init local git clone for all apps") + parser.add_argument("--apply", "-a", action="store_true", help="Apply patch on all local clones") + parser.add_argument("--diff", "-d", action="store_true", help="Inspect diff for all apps") + parser.add_argument("--push", "-p", action="store_true", help="Push and create pull requests on all apps with non-empty diff") + args = parser.parse_args() -# Init local git clone for all apps -./autopatch.py --build-cache + if not (args.cache or args.apply or args.diff or args.push): + parser.error("We required --cache, --apply, --diff or --push.") -# Apply patch in all local clones -./autopatch.py --apply explicit-php-version-in-deps - -# Inspect diff for all apps -./autopatch.py --diff - -# Push and create pull requests on all apps with non-empty diff -./autopatch.py --push explicit-php-version-in-deps -""" - ) - - elif action == "--build-cache": + if args.cache: build_cache() - elif action == "--apply": - apply(sys.argv[2]) - elif action == "--diff": - diff() - elif action == "--push": - push(sys.argv[2]) - else: - print("Unknown action %s" % action) + if args.apply: + if not args.the_patch: + parser.error("--apply requires the patch name to be passed") + apply(args.the_patch) + + if args.diff: + diff() + + if args.push: + if not args.the_patch: + parser.error("--push requires the patch name to be passed") + push(args.the_patch) main() diff --git a/tools/autoupdate_app_sources/autoupdate_app_sources.py b/tools/autoupdate_app_sources/autoupdate_app_sources.py index 1ce08cc4..93ea08b9 100755 --- a/tools/autoupdate_app_sources/autoupdate_app_sources.py +++ b/tools/autoupdate_app_sources/autoupdate_app_sources.py @@ -27,15 +27,16 @@ from rest_api import ( DownloadPageAPI, RefType, ) # noqa: E402,E501 pylint: disable=import-error,wrong-import-position +import appslib.get_apps_repo as get_apps_repo import appslib.logging_sender # noqa: E402 pylint: disable=import-error,wrong-import-position from appslib.utils import ( - REPO_APPS_ROOT, get_catalog, ) # noqa: E402 pylint: disable=import-error,wrong-import-position from app_caches import ( app_cache_folder, ) # noqa: E402 pylint: disable=import-error,wrong-import-position +TOOLS_DIR = Path(__file__).resolve().parent.parent STRATEGIES = [ "latest_github_release", @@ -62,19 +63,19 @@ def get_github() -> tuple[ ]: try: github_login = ( - (REPO_APPS_ROOT / "tools" / ".github_login") + (TOOLS_DIR / ".github_login") .open("r", encoding="utf-8") .read() .strip() ) github_token = ( - (REPO_APPS_ROOT / "tools" / ".github_token") + (TOOLS_DIR / ".github_token") .open("r", encoding="utf-8") .read() .strip() ) github_email = ( - (REPO_APPS_ROOT / "tools" / ".github_email") + (TOOLS_DIR / ".github_email") .open("r", encoding="utf-8") .read() .strip() @@ -89,10 +90,10 @@ def get_github() -> tuple[ return None, None, None -def apps_to_run_auto_update_for() -> list[str]: +def apps_to_run_auto_update_for(apps_repo: Path) -> list[str]: apps_flagged_as_working_and_on_yunohost_apps_org = [ app - for app, infos in get_catalog().items() + for app, infos in get_catalog(apps_repo).items() if infos["state"] == "working" and "/github.com/yunohost-apps" in infos["url"].lower() ] @@ -746,6 +747,7 @@ def main() -> None: parser.add_argument( "-j", "--processes", type=int, default=multiprocessing.cpu_count() ) + get_apps_repo.add_args(parser) args = parser.parse_args() appslib.logging_sender.enable() @@ -758,10 +760,12 @@ def main() -> None: sys.exit(1) # Handle apps or no apps - apps = list(args.apps) if args.apps else apps_to_run_auto_update_for() + apps = list(args.apps) if args.apps else apps_to_run_auto_update_for(get_apps_repo.from_args(args)) apps_already = {} # for which a PR already exists apps_updated = {} apps_failed = {} + print(apps) + exit() with multiprocessing.Pool(processes=args.processes) as pool: tasks = pool.imap( diff --git a/tools/catalog_linter.py b/tools/catalog_linter.py index b4438f9d..19b20ff3 100755 --- a/tools/catalog_linter.py +++ b/tools/catalog_linter.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import json import sys from pathlib import Path @@ -7,6 +8,7 @@ from difflib import SequenceMatcher from typing import Any, Dict, Generator, List, Tuple import jsonschema +import appslib.get_apps_repo as get_apps_repo from appslib.utils import ( REPO_APPS_ROOT, # pylint: disable=import-error get_antifeatures, @@ -98,6 +100,10 @@ def check_all_apps() -> Generator[Tuple[str, List[Tuple[str, bool]]], None, None def main() -> None: + parser = argparse.ArgumentParser() + get_apps_repo.add_args(parser) + args = parser.parse_args() + has_errors = False has_errors |= validate_schema_pretty(get_antifeatures(), "antifeatures.toml") diff --git a/tools/readme_generator/make_readme.py b/tools/readme_generator/make_readme.py index f608e02b..764d7442 100755 --- a/tools/readme_generator/make_readme.py +++ b/tools/readme_generator/make_readme.py @@ -1,5 +1,6 @@ #! /usr/bin/env python3 +import sys import os import argparse import json @@ -14,8 +15,12 @@ from babel.support import Translations from babel.messages.pofile import PoFileParser from langcodes import Language +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from appslib import get_apps_repo + README_GEN_DIR = Path(__file__).resolve().parent -APPS_REPO_ROOT = README_GEN_DIR.parent.parent TRANSLATIONS_DIR = README_GEN_DIR / "translations" @@ -31,7 +36,7 @@ def value_for_lang(values: Dict, lang: str): return list(values.values())[0] -def generate_READMEs(app_path: Path): +def generate_READMEs(app_path: Path, apps_repo_path: Path): if not app_path.exists(): raise Exception("App path provided doesn't exists ?!") @@ -42,11 +47,11 @@ def generate_READMEs(app_path: Path): upstream = manifest.get("upstream", {}) - catalog = toml.load((APPS_REPO_ROOT / "apps.toml").open(encoding="utf-8")) + catalog = toml.load((apps_repo_path / "apps.toml").open(encoding="utf-8")) from_catalog = catalog.get(manifest["id"], {}) antifeatures_list = toml.load( - (APPS_REPO_ROOT / "antifeatures.toml").open(encoding="utf-8") + (apps_repo_path / "antifeatures.toml").open(encoding="utf-8") ) if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists(): @@ -188,13 +193,16 @@ def generate_READMEs(app_path: Path): (app_path / "ALL_README.md").write_text(out) -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Automatically (re)generate README for apps" - ) - parser.add_argument( - "app_path", type=Path, help="Path to the app to generate/update READMEs for" - ) - +def main(): + parser = argparse.ArgumentParser(description="Automatically (re)generate README for apps") + parser.add_argument("app_path", type=Path, help="Path to the app to generate/update READMEs for") + get_apps_repo.add_args(parser) args = parser.parse_args() - generate_READMEs(Path(args.app_path)) + + apps_path = get_apps_repo.from_args(args) + + generate_READMEs(args.app_path, apps_path) + + +if __name__ == "__main__": + main() diff --git a/tools/readme_generator/tests/test_make_readme.py b/tools/readme_generator/tests/test_make_readme.py index 893204b5..6ad070c1 100755 --- a/tools/readme_generator/tests/test_make_readme.py +++ b/tools/readme_generator/tests/test_make_readme.py @@ -26,9 +26,11 @@ def test_running_make_readme(): ) # Now run test... - subprocess.check_call( - [TEST_DIRECTORY.parent / "make_readme.py", temporary_tested_app_directory] - ) + subprocess.check_call([ + TEST_DIRECTORY.parent / "make_readme.py", + "--apps-dir", TEST_DIRECTORY.parent.parent, + temporary_tested_app_directory + ]) assert ( open(TEST_DIRECTORY / "README.md").read() diff --git a/tools/save_added_date.py b/tools/save_added_date.py index f31c8331..40564864 100755 --- a/tools/save_added_date.py +++ b/tools/save_added_date.py @@ -1,22 +1,17 @@ #!/usr/bin/env python3 +import argparse import tomlkit import json from datetime import datetime from git import Repo, Commit from pathlib import Path import logging -from typing import TYPE_CHECKING, Callable - -if TYPE_CHECKING: - REPO_APPS_ROOT = Path() -else: - from appslib.utils import REPO_APPS_ROOT +from typing import Callable +import appslib.get_apps_repo as get_apps_repo -def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None: - repo = Repo(repo_path) - +def git_bisect(repo: Repo, is_newer: Callable[[Commit], bool]) -> Commit | None: # Start with whole repo first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0) repo.git.bisect("reset") @@ -69,19 +64,19 @@ def app_is_deprecated(commit: Commit, name: str) -> bool: return "deprecated-software" in antifeatures -def date_added(name: str) -> int | None: - result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name)) +def date_added(repo: Repo, name: str) -> int | None: + result = git_bisect(repo, lambda x: app_is_present(x, name)) print(result) return None if result is None else result.committed_date -def date_deprecated(name: str) -> int | None: - result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name)) +def date_deprecated(repo: Repo, name: str) -> int | None: + result = git_bisect(repo, lambda x: app_is_deprecated(x, name)) print(result) return None if result is None else result.committed_date -def add_deprecation_dates(file: Path) -> None: +def add_deprecation_dates(repo: Repo, file: Path) -> None: key = "deprecated_date" document = tomlkit.load(file.open("r", encoding="utf-8")) for app, info in document.items(): @@ -89,7 +84,7 @@ def add_deprecation_dates(file: Path) -> None: continue if "deprecated-software" not in info.get("antifeatures", []): continue - date = date_deprecated(app) + date = date_deprecated(repo, app) if date is None: continue info[key] = date @@ -98,10 +93,9 @@ def add_deprecation_dates(file: Path) -> None: tomlkit.dump(document, file.open("w")) -def date_added_to(match: str, file: Path) -> int | None: +def date_added_to(repo: Repo, match: str, file: Path) -> int | None: commits = ( - Repo(REPO_APPS_ROOT) - .git.log( + repo.git.log( "-S", match, "--first-parent", @@ -120,12 +114,12 @@ def date_added_to(match: str, file: Path) -> int | None: return int(first_commit) -def add_apparition_dates(file: Path, key: str) -> None: +def add_apparition_dates(repo: Repo, file: Path, key: str) -> None: document = tomlkit.load(file.open("r", encoding="utf-8")) for app, info in document.items(): if key in info.keys(): continue - date = date_added_to(f"[{app}]", file) + date = date_added_to(repo, f"[{app}]", file) assert date is not None info[key] = date info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d")) @@ -134,14 +128,21 @@ def add_apparition_dates(file: Path, key: str) -> None: def main() -> None: + parser = argparse.ArgumentParser() + get_apps_repo.add_args(parser, allow_temp=False) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) - add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date") - add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date") - add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date") + apps_repo_dir = get_apps_repo.from_args(args) + apps_repo = Repo(apps_repo_dir) - add_deprecation_dates(REPO_APPS_ROOT / "apps.toml") - add_deprecation_dates(REPO_APPS_ROOT / "graveyard.toml") + add_apparition_dates(apps_repo, apps_repo_dir / "apps.toml", key="added_date") + add_apparition_dates(apps_repo, apps_repo_dir / "wishlist.toml", key="added_date") + add_apparition_dates(apps_repo, apps_repo_dir / "graveyard.toml", key="killed_date") + + add_deprecation_dates(apps_repo, apps_repo_dir / "apps.toml") + add_deprecation_dates(apps_repo, apps_repo_dir / "graveyard.toml") if __name__ == "__main__": diff --git a/tools/update_app_levels/update_app_levels.py b/tools/update_app_levels/update_app_levels.py index ef192f97..f1a72c97 100755 --- a/tools/update_app_levels/update_app_levels.py +++ b/tools/update_app_levels/update_app_levels.py @@ -3,6 +3,7 @@ Update app catalog: commit, and create a merge request """ +import sys import argparse import logging import tempfile @@ -17,12 +18,16 @@ import tomlkit import tomlkit.items from git import Repo +# add apps/tools to sys.path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from appslib import get_apps_repo + APPS_REPO = "YunoHost/apps" CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/api/results" -REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir) -TOOLS_DIR = REPO_APPS_ROOT / "tools" +TOOLS_DIR = Path(__file__).resolve().parent.parent def github_token() -> Optional[str]: @@ -206,49 +211,49 @@ def main(): parser.add_argument("--commit", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--pr", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("-v", "--verbose", action=argparse.BooleanOptionalAction) + get_apps_repo.add_args(parser) args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - with tempfile.TemporaryDirectory(prefix="update_app_levels_") as tmpdir: - logging.info("Cloning the repository...") - apps_repo = Repo.clone_from(f"git@github.com:{APPS_REPO}", to_path=tmpdir) - assert apps_repo.working_tree_dir is not None - apps_toml_path = Path(apps_repo.working_tree_dir) / "apps.toml" + repo_path = get_apps_repo.from_args(args) - # Load the app catalog and filter out the non-working ones - catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8")) + apps_repo = Repo(repo_path) + apps_toml_path = repo_path / "apps.toml" - new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master) - apps_repo.head.reference = new_branch + # Load the app catalog and filter out the non-working ones + catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8")) - logging.info("Retrieving the CI results...") - ci_results = get_ci_results() + new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master) + apps_repo.head.reference = new_branch - # Now compute changes, then update the catalog - changes = list_changes(catalog, ci_results) - pr_body = pretty_changes(changes) - catalog = update_catalog(catalog, ci_results) + logging.info("Retrieving the CI results...") + ci_results = get_ci_results() - # Save the new catalog - updated_catalog = tomlkit.dumps(catalog) - updated_catalog = updated_catalog.replace(",]", " ]") - apps_toml_path.open("w", encoding="utf-8").write(updated_catalog) + # Now compute changes, then update the catalog + changes = list_changes(catalog, ci_results) + pr_body = pretty_changes(changes) + catalog = update_catalog(catalog, ci_results) - if args.commit: - logging.info("Committing and pushing the new catalog...") - apps_repo.index.add("apps.toml") - apps_repo.index.commit("Update app levels according to CI results") - apps_repo.git.push("--set-upstream", "origin", new_branch) + # Save the new catalog + updated_catalog = tomlkit.dumps(catalog) + updated_catalog = updated_catalog.replace(",]", " ]") + apps_toml_path.open("w", encoding="utf-8").write(updated_catalog) - if args.verbose: - print(pr_body) + if args.commit: + logging.info("Committing and pushing the new catalog...") + apps_repo.index.add("apps.toml") + apps_repo.index.commit("Update app levels according to CI results") + apps_repo.git.push("--set-upstream", "origin", new_branch) - if args.pr: - logging.info("Opening a pull request...") - make_pull_request(pr_body) + if args.verbose: + print(pr_body) + + if args.pr: + logging.info("Opening a pull request...") + make_pull_request(pr_body) if __name__ == "__main__":