Merge db7cd6f29e into 8c9d6219ab

tools/appslib/get_apps_repo.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+import tempfile
+import logging
+from pathlib import Path
+from typing import Optional
+from git import Repo
+
+
+DEFAULT_GIT_REPO = "git@github.com:YunoHost/apps"
+
+
+class TemporaryPath(Path):
+    """ Just a helper to return agnostically a Path or a TemporaryDirectory """
+    def __init__(self, *args, **kwargs):
+        self.temporary_directory = tempfile.TemporaryDirectory(*args, **kwargs)
+        Path.__init__(self, self.temporary_directory.name)
+
+    def with_segments(self, *pathsegments):
+        """ We need to overload this method because it calls type(self)
+            but we don't want to create multiple TemporaryPaths.
+        """
+        return Path(*pathsegments)
+
+
+APPS_REPO_DIR: Optional[TemporaryPath] = None
+
+def add_args(parser: argparse.ArgumentParser, required: bool = False, allow_temp: bool = True) -> None:
+    env_apps_dir_str = os.environ.get("YNH_APPS_DIR")
+    env_apps_dir = Path(env_apps_dir_str) if env_apps_dir_str is not None else None
+
+    repo_group = parser.add_mutually_exclusive_group(required=required)
+    if allow_temp:
+        repo_group.add_argument("-c", "--apps-repo", type=str, default=DEFAULT_GIT_REPO,
+                                help="Git url to clone the 'apps' repository")
+    repo_group.add_argument("-d", "--apps-dir", type=Path, help="Path to an existing 'apps' repository", default=env_apps_dir)
+
+
+def from_args(args: Optional[argparse.Namespace]) -> Path:
+    global APPS_REPO_DIR
+    if APPS_REPO_DIR is not None:
+        return APPS_REPO_DIR
+
+    assert args is not None
+    if args.apps_dir is not None:
+        APPS_REPO_DIR = args.apps_dir
+        assert APPS_REPO_DIR is not None
+        return APPS_REPO_DIR
+
+    if args.apps_repo is not None:
+        tmpdir = TemporaryPath(prefix="yunohost_apps_")
+        logging.info("Cloning the 'apps' repository...")
+        repo = Repo.clone_from(args.apps_repo, to_path=tmpdir)
+        assert repo.working_tree_dir is not None
+        APPS_REPO_DIR = tmpdir
+        return APPS_REPO_DIR
+
+    raise RuntimeError("You need to pass either --apps-repo or --apps-dir!")

tools/appslib/utils.py

@@ -13,11 +13,6 @@ import toml
 REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir)
 
 
-@cache
-def apps_repo_root() -> Path:
-    return Path(__file__).parent.parent.parent
-
-
 def git(cmd: list[str], cwd: Optional[Path] = None) -> str:
     full_cmd = ["git"]
     if cwd:
@@ -41,9 +36,11 @@ def git_repo_age(path: Path) -> Union[bool, int]:
 
 
 @cache
-def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]:
+def get_catalog(apps_repo: Path | None = None, working_only: bool = False) -> dict[str, dict[str, Any]]:
     """Load the app catalog and filter out the non-working ones"""
-    catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8"))
+    apps_repo = apps_repo or REPO_APPS_ROOT
+
+    catalog = toml.load((apps_repo / "apps.toml").open("r", encoding="utf-8"))
     if working_only:
         catalog = {
             app: infos

tools/autopatches/autopatch.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import json
 import os
 import subprocess
@@ -14,19 +15,20 @@ import toml
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from appslib.utils import (  # noqa: E402 pylint: disable=import-error,wrong-import-position
-    REPO_APPS_ROOT,
     get_catalog,
 )
 
+TOOLS_DIR = Path(__file__).resolve().parent.parent
+
 my_env = os.environ.copy()
 my_env["GIT_TERMINAL_PROMPT"] = "0"
 os.makedirs(".apps_cache", exist_ok=True)
 
 login = (
-    (REPO_APPS_ROOT / "tools/.github_login").open("r", encoding="utf-8").read().strip()
+    (TOOLS_DIR / ".github_login").open("r", encoding="utf-8").read().strip()
 )
 token = (
-    (REPO_APPS_ROOT / "tools/.github_token").open("r", encoding="utf-8").read().strip()
+    (TOOLS_DIR / ".github_token").open("r", encoding="utf-8").read().strip()
 )
 github_api = "https://api.github.com"
 
@@ -193,36 +195,31 @@ def create_pull_request(repo, patch, base_branch, s):
 
 
 def main():
-    action = sys.argv[1]
+    parser = argparse.ArgumentParser()
-    if action == "--help":
+    parser.add_argument("the_patch", type=str, nargs="?", help="The name of the patch to apply")
-        print(
+    parser.add_argument("--cache", "-b", action="store_true", help="Init local git clone for all apps")
-            """
+    parser.add_argument("--apply", "-a", action="store_true", help="Apply patch on all local clones")
-    Example usage:
+    parser.add_argument("--diff", "-d", action="store_true", help="Inspect diff for all apps")
+    parser.add_argument("--push", "-p", action="store_true", help="Push and create pull requests on all apps with non-empty diff")
+    args = parser.parse_args()
 
-# Init local git clone for all apps
+    if not (args.cache or args.apply or args.diff or args.push):
-./autopatch.py --build-cache
+        parser.error("We required --cache, --apply, --diff or --push.")
 
-# Apply patch in all local clones
+    if args.cache:
-./autopatch.py --apply explicit-php-version-in-deps
-
-# Inspect diff for all apps
-./autopatch.py --diff
-
-# Push and create pull requests on all apps with non-empty diff
-./autopatch.py --push explicit-php-version-in-deps
-"""
-        )
-
-    elif action == "--build-cache":
         build_cache()
-    elif action == "--apply":
-        apply(sys.argv[2])
-    elif action == "--diff":
-        diff()
-    elif action == "--push":
-        push(sys.argv[2])
-    else:
-        print("Unknown action %s" % action)
 
+    if args.apply:
+        if not args.the_patch:
+            parser.error("--apply requires the patch name to be passed")
+        apply(args.the_patch)
+
+    if args.diff:
+        diff()
+
+    if args.push:
+        if not args.the_patch:
+            parser.error("--push requires the patch name to be passed")
+        push(args.the_patch)
 
 main()

tools/autoupdate_app_sources/autoupdate_app_sources.py

@@ -27,15 +27,16 @@ from rest_api import (
     DownloadPageAPI,
     RefType,
 )  # noqa: E402,E501 pylint: disable=import-error,wrong-import-position
+import appslib.get_apps_repo as get_apps_repo
 import appslib.logging_sender  # noqa: E402 pylint: disable=import-error,wrong-import-position
 from appslib.utils import (
-    REPO_APPS_ROOT,
     get_catalog,
 )  # noqa: E402 pylint: disable=import-error,wrong-import-position
 from app_caches import (
     app_cache_folder,
 )  # noqa: E402 pylint: disable=import-error,wrong-import-position
 
+TOOLS_DIR = Path(__file__).resolve().parent.parent
 
 STRATEGIES = [
     "latest_github_release",
@@ -62,19 +63,19 @@ def get_github() -> tuple[
 ]:
     try:
         github_login = (
-            (REPO_APPS_ROOT / "tools" / ".github_login")
+            (TOOLS_DIR / ".github_login")
             .open("r", encoding="utf-8")
             .read()
             .strip()
         )
         github_token = (
-            (REPO_APPS_ROOT / "tools" / ".github_token")
+            (TOOLS_DIR / ".github_token")
             .open("r", encoding="utf-8")
             .read()
             .strip()
         )
         github_email = (
-            (REPO_APPS_ROOT / "tools" / ".github_email")
+            (TOOLS_DIR / ".github_email")
             .open("r", encoding="utf-8")
             .read()
             .strip()
@@ -89,10 +90,10 @@ def get_github() -> tuple[
         return None, None, None
 
 
-def apps_to_run_auto_update_for() -> list[str]:
+def apps_to_run_auto_update_for(apps_repo: Path) -> list[str]:
     apps_flagged_as_working_and_on_yunohost_apps_org = [
         app
-        for app, infos in get_catalog().items()
+        for app, infos in get_catalog(apps_repo).items()
         if infos["state"] == "working"
         and "/github.com/yunohost-apps" in infos["url"].lower()
     ]
@@ -746,6 +747,7 @@ def main() -> None:
     parser.add_argument(
         "-j", "--processes", type=int, default=multiprocessing.cpu_count()
     )
+    get_apps_repo.add_args(parser)
     args = parser.parse_args()
 
     appslib.logging_sender.enable()
@@ -758,10 +760,12 @@ def main() -> None:
         sys.exit(1)
 
     # Handle apps or no apps
-    apps = list(args.apps) if args.apps else apps_to_run_auto_update_for()
+    apps = list(args.apps) if args.apps else apps_to_run_auto_update_for(get_apps_repo.from_args(args))
     apps_already = {}  # for which a PR already exists
     apps_updated = {}
     apps_failed = {}
+    print(apps)
+    exit()
 
     with multiprocessing.Pool(processes=args.processes) as pool:
         tasks = pool.imap(

tools/catalog_linter.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import json
 import sys
 from pathlib import Path
@@ -7,6 +8,7 @@ from difflib import SequenceMatcher
 from typing import Any, Dict, Generator, List, Tuple
 
 import jsonschema
+import appslib.get_apps_repo as get_apps_repo
 from appslib.utils import (
     REPO_APPS_ROOT,  # pylint: disable=import-error
     get_antifeatures,
@@ -98,6 +100,10 @@ def check_all_apps() -> Generator[Tuple[str, List[Tuple[str, bool]]], None, None
 
 
 def main() -> None:
+    parser = argparse.ArgumentParser()
+    get_apps_repo.add_args(parser)
+    args = parser.parse_args()
+
     has_errors = False
 
     has_errors |= validate_schema_pretty(get_antifeatures(), "antifeatures.toml")

tools/readme_generator/make_readme.py

@@ -1,5 +1,6 @@
 #! /usr/bin/env python3
 
+import sys
 import os
 import argparse
 import json
@@ -14,8 +15,12 @@ from babel.support import Translations
 from babel.messages.pofile import PoFileParser
 from langcodes import Language
 
+# add apps/tools to sys.path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from appslib import get_apps_repo
+
 README_GEN_DIR = Path(__file__).resolve().parent
-APPS_REPO_ROOT = README_GEN_DIR.parent.parent
 
 TRANSLATIONS_DIR = README_GEN_DIR / "translations"
 
@@ -31,7 +36,7 @@ def value_for_lang(values: Dict, lang: str):
         return list(values.values())[0]
 
 
-def generate_READMEs(app_path: Path):
+def generate_READMEs(app_path: Path, apps_repo_path: Path):
     if not app_path.exists():
         raise Exception("App path provided doesn't exists ?!")
 
@@ -42,11 +47,11 @@ def generate_READMEs(app_path: Path):
 
     upstream = manifest.get("upstream", {})
 
-    catalog = toml.load((APPS_REPO_ROOT / "apps.toml").open(encoding="utf-8"))
+    catalog = toml.load((apps_repo_path / "apps.toml").open(encoding="utf-8"))
     from_catalog = catalog.get(manifest["id"], {})
 
     antifeatures_list = toml.load(
-        (APPS_REPO_ROOT / "antifeatures.toml").open(encoding="utf-8")
+        (apps_repo_path / "antifeatures.toml").open(encoding="utf-8")
     )
 
     if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists():
@@ -188,13 +193,16 @@ def generate_READMEs(app_path: Path):
     (app_path / "ALL_README.md").write_text(out)
 
 
-if __name__ == "__main__":
+def main():
-    parser = argparse.ArgumentParser(
+    parser = argparse.ArgumentParser(description="Automatically (re)generate README for apps")
-        description="Automatically (re)generate README for apps"
+    parser.add_argument("app_path", type=Path, help="Path to the app to generate/update READMEs for")
-    )
+    get_apps_repo.add_args(parser)
-    parser.add_argument(
-        "app_path", type=Path, help="Path to the app to generate/update READMEs for"
-    )
-
     args = parser.parse_args()
-    generate_READMEs(Path(args.app_path))
+
+    apps_path = get_apps_repo.from_args(args)
+
+    generate_READMEs(args.app_path, apps_path)
+
+
+if __name__ == "__main__":
+    main()

tools/readme_generator/tests/test_make_readme.py

@@ -26,9 +26,11 @@ def test_running_make_readme():
         )
 
         # Now run test...
-        subprocess.check_call(
+        subprocess.check_call([
-            [TEST_DIRECTORY.parent / "make_readme.py", temporary_tested_app_directory]
+            TEST_DIRECTORY.parent / "make_readme.py",
-        )
+            "--apps-dir", TEST_DIRECTORY.parent.parent,
+            temporary_tested_app_directory
+        ])
 
         assert (
             open(TEST_DIRECTORY / "README.md").read()

tools/save_added_date.py

@@ -1,22 +1,17 @@
 #!/usr/bin/env python3
 
+import argparse
 import tomlkit
 import json
 from datetime import datetime
 from git import Repo, Commit
 from pathlib import Path
 import logging
-from typing import TYPE_CHECKING, Callable
+from typing import Callable
-
+import appslib.get_apps_repo as get_apps_repo
-if TYPE_CHECKING:
-    REPO_APPS_ROOT = Path()
-else:
-    from appslib.utils import REPO_APPS_ROOT
 
 
-def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None:
+def git_bisect(repo: Repo, is_newer: Callable[[Commit], bool]) -> Commit | None:
-    repo = Repo(repo_path)
-
     # Start with whole repo
     first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0)
     repo.git.bisect("reset")
@@ -69,19 +64,19 @@ def app_is_deprecated(commit: Commit, name: str) -> bool:
     return "deprecated-software" in antifeatures
 
 
-def date_added(name: str) -> int | None:
+def date_added(repo: Repo, name: str) -> int | None:
-    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name))
+    result = git_bisect(repo, lambda x: app_is_present(x, name))
     print(result)
     return None if result is None else result.committed_date
 
 
-def date_deprecated(name: str) -> int | None:
+def date_deprecated(repo: Repo, name: str) -> int | None:
-    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name))
+    result = git_bisect(repo, lambda x: app_is_deprecated(x, name))
     print(result)
     return None if result is None else result.committed_date
 
 
-def add_deprecation_dates(file: Path) -> None:
+def add_deprecation_dates(repo: Repo, file: Path) -> None:
     key = "deprecated_date"
     document = tomlkit.load(file.open("r", encoding="utf-8"))
     for app, info in document.items():
@@ -89,7 +84,7 @@ def add_deprecation_dates(file: Path) -> None:
             continue
         if "deprecated-software" not in info.get("antifeatures", []):
             continue
-        date = date_deprecated(app)
+        date = date_deprecated(repo, app)
         if date is None:
             continue
         info[key] = date
@@ -98,10 +93,9 @@ def add_deprecation_dates(file: Path) -> None:
     tomlkit.dump(document, file.open("w"))
 
 
-def date_added_to(match: str, file: Path) -> int | None:
+def date_added_to(repo: Repo, match: str, file: Path) -> int | None:
     commits = (
-        Repo(REPO_APPS_ROOT)
+        repo.git.log(
-        .git.log(
             "-S",
             match,
             "--first-parent",
@@ -120,12 +114,12 @@ def date_added_to(match: str, file: Path) -> int | None:
     return int(first_commit)
 
 
-def add_apparition_dates(file: Path, key: str) -> None:
+def add_apparition_dates(repo: Repo, file: Path, key: str) -> None:
     document = tomlkit.load(file.open("r", encoding="utf-8"))
     for app, info in document.items():
         if key in info.keys():
             continue
-        date = date_added_to(f"[{app}]", file)
+        date = date_added_to(repo, f"[{app}]", file)
         assert date is not None
         info[key] = date
         info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
@@ -134,14 +128,21 @@ def add_apparition_dates(file: Path, key: str) -> None:
 
 
 def main() -> None:
+    parser = argparse.ArgumentParser()
+    get_apps_repo.add_args(parser, allow_temp=False)
+    args = parser.parse_args()
+
     logging.basicConfig(level=logging.DEBUG)
 
-    add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date")
+    apps_repo_dir = get_apps_repo.from_args(args)
-    add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date")
+    apps_repo = Repo(apps_repo_dir)
-    add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date")
 
-    add_deprecation_dates(REPO_APPS_ROOT / "apps.toml")
+    add_apparition_dates(apps_repo, apps_repo_dir / "apps.toml", key="added_date")
-    add_deprecation_dates(REPO_APPS_ROOT / "graveyard.toml")
+    add_apparition_dates(apps_repo, apps_repo_dir / "wishlist.toml", key="added_date")
+    add_apparition_dates(apps_repo, apps_repo_dir / "graveyard.toml", key="killed_date")
+
+    add_deprecation_dates(apps_repo, apps_repo_dir / "apps.toml")
+    add_deprecation_dates(apps_repo, apps_repo_dir / "graveyard.toml")
 
 
 if __name__ == "__main__":

tools/update_app_levels/update_app_levels.py

@@ -3,6 +3,7 @@
 Update app catalog: commit, and create a merge request
 """
 
+import sys
 import argparse
 import logging
 import tempfile
@@ -17,12 +18,16 @@ import tomlkit
 import tomlkit.items
 from git import Repo
 
+# add apps/tools to sys.path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from appslib import get_apps_repo
+
 APPS_REPO = "YunoHost/apps"
 
 CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/api/results"
 
-REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir)
+TOOLS_DIR = Path(__file__).resolve().parent.parent
-TOOLS_DIR = REPO_APPS_ROOT / "tools"
 
 
 def github_token() -> Optional[str]:
@@ -206,49 +211,49 @@ def main():
     parser.add_argument("--commit", action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--pr", action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("-v", "--verbose", action=argparse.BooleanOptionalAction)
+    get_apps_repo.add_args(parser)
     args = parser.parse_args()
 
     logging.getLogger().setLevel(logging.INFO)
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
 
-    with tempfile.TemporaryDirectory(prefix="update_app_levels_") as tmpdir:
+    repo_path = get_apps_repo.from_args(args)
-        logging.info("Cloning the repository...")
-        apps_repo = Repo.clone_from(f"git@github.com:{APPS_REPO}", to_path=tmpdir)
-        assert apps_repo.working_tree_dir is not None
-        apps_toml_path = Path(apps_repo.working_tree_dir) / "apps.toml"
 
-        # Load the app catalog and filter out the non-working ones
+    apps_repo = Repo(repo_path)
-        catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8"))
+    apps_toml_path = repo_path / "apps.toml"
 
-        new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master)
+    # Load the app catalog and filter out the non-working ones
-        apps_repo.head.reference = new_branch
+    catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8"))
 
-        logging.info("Retrieving the CI results...")
+    new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master)
-        ci_results = get_ci_results()
+    apps_repo.head.reference = new_branch
 
-        # Now compute changes, then update the catalog
+    logging.info("Retrieving the CI results...")
-        changes = list_changes(catalog, ci_results)
+    ci_results = get_ci_results()
-        pr_body = pretty_changes(changes)
-        catalog = update_catalog(catalog, ci_results)
 
-        # Save the new catalog
+    # Now compute changes, then update the catalog
-        updated_catalog = tomlkit.dumps(catalog)
+    changes = list_changes(catalog, ci_results)
-        updated_catalog = updated_catalog.replace(",]", " ]")
+    pr_body = pretty_changes(changes)
-        apps_toml_path.open("w", encoding="utf-8").write(updated_catalog)
+    catalog = update_catalog(catalog, ci_results)
 
-        if args.commit:
+    # Save the new catalog
-            logging.info("Committing and pushing the new catalog...")
+    updated_catalog = tomlkit.dumps(catalog)
-            apps_repo.index.add("apps.toml")
+    updated_catalog = updated_catalog.replace(",]", " ]")
-            apps_repo.index.commit("Update app levels according to CI results")
+    apps_toml_path.open("w", encoding="utf-8").write(updated_catalog)
-            apps_repo.git.push("--set-upstream", "origin", new_branch)
 
-        if args.verbose:
+    if args.commit:
-            print(pr_body)
+        logging.info("Committing and pushing the new catalog...")
+        apps_repo.index.add("apps.toml")
+        apps_repo.index.commit("Update app levels according to CI results")
+        apps_repo.git.push("--set-upstream", "origin", new_branch)
 
-        if args.pr:
+    if args.verbose:
-            logging.info("Opening a pull request...")
+        print(pr_body)
-            make_pull_request(pr_body)
+
+    if args.pr:
+        logging.info("Opening a pull request...")
+        make_pull_request(pr_body)
 
 
 if __name__ == "__main__":