YunoHost/apps
1
0
Fork 0
mirror of https://github.com/YunoHost/apps.git synced 2024-09-03 20:06:07 +02:00
Code Projects Activity

Merge db7cd6f29e into 8c9d6219ab

Browse source
This commit is contained in:
Salamandar 2024-08-31 09:48:30 +02:00 • committed by GitHub
commit 28b3954de1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 196 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
tools/appslib/get_apps_repo.py Normal file
View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
import os
import argparse
import tempfile
import logging
from pathlib import Path
from typing import Optional
from git import Repo
DEFAULT_GIT_REPO = "git@github.com:YunoHost/apps"
class TemporaryPath(Path):
""" Just a helper to return agnostically a Path or a TemporaryDirectory """
def __init__(self, *args, **kwargs):
self.temporary_directory = tempfile.TemporaryDirectory(*args, **kwargs)
Path.__init__(self, self.temporary_directory.name)
def with_segments(self, *pathsegments):
""" We need to overload this method because it calls type(self)
but we don't want to create multiple TemporaryPaths.
"""
return Path(*pathsegments)
APPS_REPO_DIR: Optional[TemporaryPath] = None
def add_args(parser: argparse.ArgumentParser, required: bool = False, allow_temp: bool = True) -> None:
env_apps_dir_str = os.environ.get("YNH_APPS_DIR")
env_apps_dir = Path(env_apps_dir_str) if env_apps_dir_str is not None else None
repo_group = parser.add_mutually_exclusive_group(required=required)
if allow_temp:
repo_group.add_argument("-c", "--apps-repo", type=str, default=DEFAULT_GIT_REPO,
help="Git url to clone the 'apps' repository")
repo_group.add_argument("-d", "--apps-dir", type=Path, help="Path to an existing 'apps' repository", default=env_apps_dir)
def from_args(args: Optional[argparse.Namespace]) -> Path:
global APPS_REPO_DIR
if APPS_REPO_DIR is not None:
return APPS_REPO_DIR
assert args is not None
if args.apps_dir is not None:
APPS_REPO_DIR = args.apps_dir
assert APPS_REPO_DIR is not None
return APPS_REPO_DIR
if args.apps_repo is not None:
tmpdir = TemporaryPath(prefix="yunohost_apps_")
logging.info("Cloning the 'apps' repository...")
repo = Repo.clone_from(args.apps_repo, to_path=tmpdir)
assert repo.working_tree_dir is not None
APPS_REPO_DIR = tmpdir
return APPS_REPO_DIR
raise RuntimeError("You need to pass either --apps-repo or --apps-dir!")

11
tools/appslib/utils.py
View file

@ -13,11 +13,6 @@ import toml
REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir)
@cache
def apps_repo_root() -> Path:
return Path(__file__).parent.parent.parent
def git(cmd: list[str], cwd: Optional[Path] = None) -> str:
full_cmd = ["git"]
if cwd:
@ -41,9 +36,11 @@ def git_repo_age(path: Path) -> Union[bool, int]:
@cache
def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]:
def get_catalog(apps_repo: Path | None = None, working_only: bool = False) -> dict[str, dict[str, Any]]:
"""Load the app catalog and filter out the non-working ones"""
catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8"))
apps_repo = apps_repo or REPO_APPS_ROOT
catalog = toml.load((apps_repo / "apps.toml").open("r", encoding="utf-8"))
if working_only:
catalog = {
app: infos

57
tools/autopatches/autopatch.py
View file

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
@ -14,19 +15,20 @@ import toml
sys.path.insert(0, str(Path(__file__).parent.parent))
from appslib.utils import ( # noqa: E402 pylint: disable=import-error,wrong-import-position
REPO_APPS_ROOT,
get_catalog,
)
TOOLS_DIR = Path(__file__).resolve().parent.parent
my_env = os.environ.copy()
my_env["GIT_TERMINAL_PROMPT"] = "0"
os.makedirs(".apps_cache", exist_ok=True)
login = (
(REPO_APPS_ROOT / "tools/.github_login").open("r", encoding="utf-8").read().strip()
(TOOLS_DIR / ".github_login").open("r", encoding="utf-8").read().strip()
)
token = (
(REPO_APPS_ROOT / "tools/.github_token").open("r", encoding="utf-8").read().strip()
(TOOLS_DIR / ".github_token").open("r", encoding="utf-8").read().strip()
)
github_api = "https://api.github.com"
@ -193,36 +195,31 @@ def create_pull_request(repo, patch, base_branch, s):
def main():
action = sys.argv[1]
if action == "--help":
print(
"""
Example usage:
parser = argparse.ArgumentParser()
parser.add_argument("the_patch", type=str, nargs="?", help="The name of the patch to apply")
parser.add_argument("--cache", "-b", action="store_true", help="Init local git clone for all apps")
parser.add_argument("--apply", "-a", action="store_true", help="Apply patch on all local clones")
parser.add_argument("--diff", "-d", action="store_true", help="Inspect diff for all apps")
parser.add_argument("--push", "-p", action="store_true", help="Push and create pull requests on all apps with non-empty diff")
args = parser.parse_args()
# Init local git clone for all apps
./autopatch.py --build-cache
if not (args.cache or args.apply or args.diff or args.push):
parser.error("We required --cache, --apply, --diff or --push.")
# Apply patch in all local clones
./autopatch.py --apply explicit-php-version-in-deps
# Inspect diff for all apps
./autopatch.py --diff
# Push and create pull requests on all apps with non-empty diff
./autopatch.py --push explicit-php-version-in-deps
"""
)
elif action == "--build-cache":
if args.cache:
build_cache()
elif action == "--apply":
apply(sys.argv[2])
elif action == "--diff":
diff()
elif action == "--push":
push(sys.argv[2])
else:
print("Unknown action %s" % action)
if args.apply:
if not args.the_patch:
parser.error("--apply requires the patch name to be passed")
apply(args.the_patch)
if args.diff:
diff()
if args.push:
if not args.the_patch:
parser.error("--push requires the patch name to be passed")
push(args.the_patch)
main()

18
tools/autoupdate_app_sources/autoupdate_app_sources.py
View file

@ -27,15 +27,16 @@ from rest_api import (
DownloadPageAPI,
RefType,
) # noqa: E402,E501 pylint: disable=import-error,wrong-import-position
import appslib.get_apps_repo as get_apps_repo
import appslib.logging_sender # noqa: E402 pylint: disable=import-error,wrong-import-position
from appslib.utils import (
REPO_APPS_ROOT,
get_catalog,
) # noqa: E402 pylint: disable=import-error,wrong-import-position
from app_caches import (
app_cache_folder,
) # noqa: E402 pylint: disable=import-error,wrong-import-position
TOOLS_DIR = Path(__file__).resolve().parent.parent
STRATEGIES = [
"latest_github_release",
@ -62,19 +63,19 @@ def get_github() -> tuple[
]:
try:
github_login = (
(REPO_APPS_ROOT / "tools" / ".github_login")
(TOOLS_DIR / ".github_login")
.open("r", encoding="utf-8")
.read()
.strip()
)
github_token = (
(REPO_APPS_ROOT / "tools" / ".github_token")
(TOOLS_DIR / ".github_token")
.open("r", encoding="utf-8")
.read()
.strip()
)
github_email = (
(REPO_APPS_ROOT / "tools" / ".github_email")
(TOOLS_DIR / ".github_email")
.open("r", encoding="utf-8")
.read()
.strip()
@ -89,10 +90,10 @@ def get_github() -> tuple[
return None, None, None
def apps_to_run_auto_update_for() -> list[str]:
def apps_to_run_auto_update_for(apps_repo: Path) -> list[str]:
apps_flagged_as_working_and_on_yunohost_apps_org = [
app
for app, infos in get_catalog().items()
for app, infos in get_catalog(apps_repo).items()
if infos["state"] == "working"
and "/github.com/yunohost-apps" in infos["url"].lower()
]
@ -746,6 +747,7 @@ def main() -> None:
parser.add_argument(
"-j", "--processes", type=int, default=multiprocessing.cpu_count()
)
get_apps_repo.add_args(parser)
args = parser.parse_args()
appslib.logging_sender.enable()
@ -758,10 +760,12 @@ def main() -> None:
sys.exit(1)
# Handle apps or no apps
apps = list(args.apps) if args.apps else apps_to_run_auto_update_for()
apps = list(args.apps) if args.apps else apps_to_run_auto_update_for(get_apps_repo.from_args(args))
apps_already = {} # for which a PR already exists
apps_updated = {}
apps_failed = {}
print(apps)
exit()
with multiprocessing.Pool(processes=args.processes) as pool:
tasks = pool.imap(

6
tools/catalog_linter.py
View file

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import json
import sys
from pathlib import Path
@ -7,6 +8,7 @@ from difflib import SequenceMatcher
from typing import Any, Dict, Generator, List, Tuple
import jsonschema
import appslib.get_apps_repo as get_apps_repo
from appslib.utils import (
REPO_APPS_ROOT, # pylint: disable=import-error
get_antifeatures,
@ -98,6 +100,10 @@ def check_all_apps() -> Generator[Tuple[str, List[Tuple[str, bool]]], None, None
def main() -> None:
parser = argparse.ArgumentParser()
get_apps_repo.add_args(parser)
args = parser.parse_args()
has_errors = False
has_errors |= validate_schema_pretty(get_antifeatures(), "antifeatures.toml")

34
tools/readme_generator/make_readme.py
View file

@ -1,5 +1,6 @@
#! /usr/bin/env python3
import sys
import os
import argparse
import json
@ -14,8 +15,12 @@ from babel.support import Translations
from babel.messages.pofile import PoFileParser
from langcodes import Language
# add apps/tools to sys.path
sys.path.insert(0, str(Path(__file__).parent.parent))
from appslib import get_apps_repo
README_GEN_DIR = Path(__file__).resolve().parent
APPS_REPO_ROOT = README_GEN_DIR.parent.parent
TRANSLATIONS_DIR = README_GEN_DIR / "translations"
@ -31,7 +36,7 @@ def value_for_lang(values: Dict, lang: str):
return list(values.values())[0]
def generate_READMEs(app_path: Path):
def generate_READMEs(app_path: Path, apps_repo_path: Path):
if not app_path.exists():
raise Exception("App path provided doesn't exists ?!")
@ -42,11 +47,11 @@ def generate_READMEs(app_path: Path):
upstream = manifest.get("upstream", {})
catalog = toml.load((APPS_REPO_ROOT / "apps.toml").open(encoding="utf-8"))
catalog = toml.load((apps_repo_path / "apps.toml").open(encoding="utf-8"))
from_catalog = catalog.get(manifest["id"], {})
antifeatures_list = toml.load(
(APPS_REPO_ROOT / "antifeatures.toml").open(encoding="utf-8")
(apps_repo_path / "antifeatures.toml").open(encoding="utf-8")
)
if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists():
@ -188,13 +193,16 @@ def generate_READMEs(app_path: Path):
(app_path / "ALL_README.md").write_text(out)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Automatically (re)generate README for apps"
)
parser.add_argument(
"app_path", type=Path, help="Path to the app to generate/update READMEs for"
)
def main():
parser = argparse.ArgumentParser(description="Automatically (re)generate README for apps")
parser.add_argument("app_path", type=Path, help="Path to the app to generate/update READMEs for")
get_apps_repo.add_args(parser)
args = parser.parse_args()
generate_READMEs(Path(args.app_path))
apps_path = get_apps_repo.from_args(args)
generate_READMEs(args.app_path, apps_path)
if __name__ == "__main__":
main()

8
tools/readme_generator/tests/test_make_readme.py
View file

@ -26,9 +26,11 @@ def test_running_make_readme():
)
# Now run test...
subprocess.check_call(
[TEST_DIRECTORY.parent / "make_readme.py", temporary_tested_app_directory]
)
subprocess.check_call([
TEST_DIRECTORY.parent / "make_readme.py",
"--apps-dir", TEST_DIRECTORY.parent.parent,
temporary_tested_app_directory
])
assert (
open(TEST_DIRECTORY / "README.md").read()

51
tools/save_added_date.py
View file

@ -1,22 +1,17 @@
#!/usr/bin/env python3
import argparse
import tomlkit
import json
from datetime import datetime
from git import Repo, Commit
from pathlib import Path
import logging
from typing import TYPE_CHECKING, Callable
if TYPE_CHECKING:
REPO_APPS_ROOT = Path()
else:
from appslib.utils import REPO_APPS_ROOT
from typing import Callable
import appslib.get_apps_repo as get_apps_repo
def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None:
repo = Repo(repo_path)
def git_bisect(repo: Repo, is_newer: Callable[[Commit], bool]) -> Commit | None:
# Start with whole repo
first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0)
repo.git.bisect("reset")
@ -69,19 +64,19 @@ def app_is_deprecated(commit: Commit, name: str) -> bool:
return "deprecated-software" in antifeatures
def date_added(name: str) -> int | None:
result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name))
def date_added(repo: Repo, name: str) -> int | None:
result = git_bisect(repo, lambda x: app_is_present(x, name))
print(result)
return None if result is None else result.committed_date
def date_deprecated(name: str) -> int | None:
result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name))
def date_deprecated(repo: Repo, name: str) -> int | None:
result = git_bisect(repo, lambda x: app_is_deprecated(x, name))
print(result)
return None if result is None else result.committed_date
def add_deprecation_dates(file: Path) -> None:
def add_deprecation_dates(repo: Repo, file: Path) -> None:
key = "deprecated_date"
document = tomlkit.load(file.open("r", encoding="utf-8"))
for app, info in document.items():
@ -89,7 +84,7 @@ def add_deprecation_dates(file: Path) -> None:
continue
if "deprecated-software" not in info.get("antifeatures", []):
continue
date = date_deprecated(app)
date = date_deprecated(repo, app)
if date is None:
continue
info[key] = date
@ -98,10 +93,9 @@ def add_deprecation_dates(file: Path) -> None:
tomlkit.dump(document, file.open("w"))
def date_added_to(match: str, file: Path) -> int | None:
def date_added_to(repo: Repo, match: str, file: Path) -> int | None:
commits = (
Repo(REPO_APPS_ROOT)
.git.log(
repo.git.log(
"-S",
match,
"--first-parent",
@ -120,12 +114,12 @@ def date_added_to(match: str, file: Path) -> int | None:
return int(first_commit)
def add_apparition_dates(file: Path, key: str) -> None:
def add_apparition_dates(repo: Repo, file: Path, key: str) -> None:
document = tomlkit.load(file.open("r", encoding="utf-8"))
for app, info in document.items():
if key in info.keys():
continue
date = date_added_to(f"[{app}]", file)
date = date_added_to(repo, f"[{app}]", file)
assert date is not None
info[key] = date
info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
@ -134,14 +128,21 @@ def add_apparition_dates(file: Path, key: str) -> None:
def main() -> None:
parser = argparse.ArgumentParser()
get_apps_repo.add_args(parser, allow_temp=False)
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date")
add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date")
add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date")
apps_repo_dir = get_apps_repo.from_args(args)
apps_repo = Repo(apps_repo_dir)
add_deprecation_dates(REPO_APPS_ROOT / "apps.toml")
add_deprecation_dates(REPO_APPS_ROOT / "graveyard.toml")
add_apparition_dates(apps_repo, apps_repo_dir / "apps.toml", key="added_date")
add_apparition_dates(apps_repo, apps_repo_dir / "wishlist.toml", key="added_date")
add_apparition_dates(apps_repo, apps_repo_dir / "graveyard.toml", key="killed_date")
add_deprecation_dates(apps_repo, apps_repo_dir / "apps.toml")
add_deprecation_dates(apps_repo, apps_repo_dir / "graveyard.toml")
if __name__ == "__main__":

67
tools/update_app_levels/update_app_levels.py
View file

@ -3,6 +3,7 @@
Update app catalog: commit, and create a merge request
"""
import sys
import argparse
import logging
import tempfile
@ -17,12 +18,16 @@ import tomlkit
import tomlkit.items
from git import Repo
# add apps/tools to sys.path
sys.path.insert(0, str(Path(__file__).parent.parent))
from appslib import get_apps_repo
APPS_REPO = "YunoHost/apps"
CI_RESULTS_URL = "https://ci-apps.yunohost.org/ci/api/results"
REPO_APPS_ROOT = Path(Repo(__file__, search_parent_directories=True).working_dir)
TOOLS_DIR = REPO_APPS_ROOT / "tools"
TOOLS_DIR = Path(__file__).resolve().parent.parent
def github_token() -> Optional[str]:
@ -206,49 +211,49 @@ def main():
parser.add_argument("--commit", action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--pr", action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-v", "--verbose", action=argparse.BooleanOptionalAction)
get_apps_repo.add_args(parser)
args = parser.parse_args()
logging.getLogger().setLevel(logging.INFO)
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
with tempfile.TemporaryDirectory(prefix="update_app_levels_") as tmpdir:
logging.info("Cloning the repository...")
apps_repo = Repo.clone_from(f"git@github.com:{APPS_REPO}", to_path=tmpdir)
assert apps_repo.working_tree_dir is not None
apps_toml_path = Path(apps_repo.working_tree_dir) / "apps.toml"
repo_path = get_apps_repo.from_args(args)
# Load the app catalog and filter out the non-working ones
catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8"))
apps_repo = Repo(repo_path)
apps_toml_path = repo_path / "apps.toml"
new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master)
apps_repo.head.reference = new_branch
# Load the app catalog and filter out the non-working ones
catalog = tomlkit.load(apps_toml_path.open("r", encoding="utf-8"))
logging.info("Retrieving the CI results...")
ci_results = get_ci_results()
new_branch = apps_repo.create_head("update_app_levels", apps_repo.refs.master)
apps_repo.head.reference = new_branch
# Now compute changes, then update the catalog
changes = list_changes(catalog, ci_results)
pr_body = pretty_changes(changes)
catalog = update_catalog(catalog, ci_results)
logging.info("Retrieving the CI results...")
ci_results = get_ci_results()
# Save the new catalog
updated_catalog = tomlkit.dumps(catalog)
updated_catalog = updated_catalog.replace(",]", " ]")
apps_toml_path.open("w", encoding="utf-8").write(updated_catalog)
# Now compute changes, then update the catalog
changes = list_changes(catalog, ci_results)
pr_body = pretty_changes(changes)
catalog = update_catalog(catalog, ci_results)
if args.commit:
logging.info("Committing and pushing the new catalog...")
apps_repo.index.add("apps.toml")
apps_repo.index.commit("Update app levels according to CI results")
apps_repo.git.push("--set-upstream", "origin", new_branch)
# Save the new catalog
updated_catalog = tomlkit.dumps(catalog)
updated_catalog = updated_catalog.replace(",]", " ]")
apps_toml_path.open("w", encoding="utf-8").write(updated_catalog)
if args.verbose:
print(pr_body)
if args.commit:
logging.info("Committing and pushing the new catalog...")
apps_repo.index.add("apps.toml")
apps_repo.index.commit("Update app levels according to CI results")
apps_repo.git.push("--set-upstream", "origin", new_branch)
if args.pr:
logging.info("Opening a pull request...")
make_pull_request(pr_body)
if args.verbose:
print(pr_body)
if args.pr:
logging.info("Opening a pull request...")
make_pull_request(pr_body)
if __name__ == "__main__":