From 5f78d99af1c4bed50df39e439a91bcfb0686d7b8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 9 Jun 2020 19:09:27 +0200 Subject: [PATCH 1/7] Draft for an autopatch 'framework' --- autopatches/autopatch.py | 191 ++++++++++++++++++ .../explicit-php-version-in-deps/patch.sh | 12 ++ .../explicit-php-version-in-deps/pr_body.md | 1 + .../explicit-php-version-in-deps/pr_title.md | 1 + 4 files changed, 205 insertions(+) create mode 100755 autopatches/autopatch.py create mode 100644 autopatches/patches/explicit-php-version-in-deps/patch.sh create mode 100644 autopatches/patches/explicit-php-version-in-deps/pr_body.md create mode 100644 autopatches/patches/explicit-php-version-in-deps/pr_title.md diff --git a/autopatches/autopatch.py b/autopatches/autopatch.py new file mode 100755 index 00000000..84662cff --- /dev/null +++ b/autopatches/autopatch.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 +import json +import sys +import requests +import os +import subprocess + +# ./autopatch.py --rebuild-cache +# ./autopatch.py --apply my_patch +# ./autopatch.py --diff +# ./autopatch.py --push my_patch + +catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() + +my_env = os.environ.copy() +my_env["GIT_TERMINAL_PROMPT"] = "0" +os.makedirs(".apps_cache", exist_ok=True) + +login = open("login").read().strip() +token = open("token").read().strip() +github_api = "https://api.github.com" + + +def apps(min_level=4): + + for app, infos in catalog.items(): + if infos.get("state") == "working" and infos.get("level", -1) > min_level: + infos["id"] = app + yield infos + + +def app_cache_folder(app): + return os.path.join(".apps_cache", app) + + +def git(cmd, in_folder=None): + + if not isinstance(cmd, list): + cmd = cmd.split() + if in_folder: + cmd = ["-C", in_folder] + cmd + cmd = ["git"] + cmd + return subprocess.check_output(cmd, env=my_env).strip().decode("utf-8") + + +# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 +def progressbar(it, prefix="", size=60, file=sys.stdout): + it = list(it) + count = len(it) + def show(j, name=""): + name += " " + x = int(size*j/count) + file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name)) + file.flush() + show(0) + for i, item in enumerate(it): + yield item + show(i+1, item["id"]) + file.write("\n") + file.flush() + + +def build_cache(): + + for app in progressbar(apps(), "Git cloning: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + reponame = app["url"].rsplit("/", 1) + git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}") + git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder) + + +def apply(patch): + + patch_path = os.path.abspath(os.path.join("patches", patch, "patch.sh")) + + for app in progressbar(apps(), "Apply to: ", 40): + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset --hard origin/{current_branch}", in_folder=folder) + os.system(f"cd {folder} && bash {patch_path}") + + +def diff(): + + for app in apps(): + print(app["id"]) + print("----------------------------") + folder = os.path.join(".apps_cache", app["id"]) + os.system(f"cd {folder} && git diff && echo $?") + + +def push(patch): + + title = "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip() + + def diff_not_empty(app): + folder = os.path.join(".apps_cache", app["id"]) + return bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")) + + def app_is_on_github(app): + return "github.com" in app["url"] + + apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)] + + apps_to_push = [app for app in apps() if app["id"] == "piwigo"] + + with requests.Session() as s: + s.headers.update({"Authorization": f"token {token}"}) + for app in progressbar(apps_to_push, "Forking: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + fork_if_needed(app["repo"], s) + + for app in progressbar(apps_to_push, "Pushing: ", 40): + app["repo"] = app["url"][len("https://github.com/"):].strip("/") + app_repo_name = app["url"].rsplit("/", 1)[-1] + folder = os.path.join(".apps_cache", app["id"]) + current_branch = git(f"symbolic-ref --short HEAD", in_folder=folder) + git(f"reset origin/{current_branch}", in_folder=folder) + git(["commit", "-a", "-m", title, "--author='Yunohost-Bot <>'"], in_folder=folder) + try: + git(f"remote remove fork", in_folder=folder) + except Exception: + pass + git(f"remote add fork https://{login}:{token}@github.com/{login}/{app_repo_name}", in_folder=folder) + git(f"push fork {current_branch}:{patch} --quiet --force", in_folder=folder) + create_pull_request(app["repo"], patch, current_branch, s) + + +def fork_if_needed(repo, s): + + repo_name = repo.split("/")[-1] + r = s.get(github_api + f"/repos/{login}/{repo_name}") + + if r.status_code == 200: + return + + r = s.post(github_api + f"/repos/{repo}/forks") + + if r.status_code != 200: + print(r.text) + + +def create_pull_request(repo, patch, base_branch, s): + + PR = {"title": "[autopatch] " + open(os.path.join("patches", patch, "pr_title.md")).read().strip(), + "body": "This is an automatic PR\n\n" + open(os.path.join("patches", patch, "pr_body.md")).read().strip(), + "head": login + ":" + patch, + "base": base_branch, + "maintainer_can_modify": True} + + r = s.post(github_api + f"/repos/{repo}/pulls", json.dumps(PR)) + + if r.status_code != 200: + print(r.text) + else: + json.loads(r.text)["html_url"] + + +def main(): + + action = sys.argv[1] + if action == "--help": + print(""" + Example usage: + +# Init local git clone for all apps +./autopatch --build-cache + +# Apply patch in all local clones +./autopatch --apply explicit-php-version-in-deps + +# Inspect diff for all apps +./autopatch --diff + +# Push and create pull requests on all apps with non-empty diff +./autopatch --push explicit-php-version-in-deps +""") + + elif action == "--build-cache": + build_cache() + elif action == "--apply": + apply(sys.argv[2]) + elif action == "--diff": + diff() + elif action == "--push": + push(sys.argv[2]) + else: + print("Unknown action %s" % action) + + +main() diff --git a/autopatches/patches/explicit-php-version-in-deps/patch.sh b/autopatches/patches/explicit-php-version-in-deps/patch.sh new file mode 100644 index 00000000..cddbb44a --- /dev/null +++ b/autopatches/patches/explicit-php-version-in-deps/patch.sh @@ -0,0 +1,12 @@ +PHP_DEPS="php-bcmath php-cli php-curl php-dev php-gd php-gmp php-imap php-intl php-json php-ldap php-mbstring php-mysql php-soap php-sqlite3 php-tidy php-xml php-xmlrpc php-zip php-dom php-opcache php-xsl php-apcu php-geoip php-imagick php-memcached php-redis php-ssh2 php-common" + +grep -q -nr "php-" scripts/* || exit 0 + +for DEP in $PHP_DEPS +do + NEWDEP=$(echo $DEP | sed 's/php-/php7.0-/g') + [ ! -e ./scripts/_common.sh ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/_common.sh + [ ! -e ./scripts/install ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/install + [ ! -e ./scripts/upgrade ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/upgrade + [ ! -e ./scripts/restore ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/restore +done diff --git a/autopatches/patches/explicit-php-version-in-deps/pr_body.md b/autopatches/patches/explicit-php-version-in-deps/pr_body.md new file mode 100644 index 00000000..2d8caed3 --- /dev/null +++ b/autopatches/patches/explicit-php-version-in-deps/pr_body.md @@ -0,0 +1 @@ +(This a test for now, please ignore, sorry for the noise) diff --git a/autopatches/patches/explicit-php-version-in-deps/pr_title.md b/autopatches/patches/explicit-php-version-in-deps/pr_title.md new file mode 100644 index 00000000..8b7f6417 --- /dev/null +++ b/autopatches/patches/explicit-php-version-in-deps/pr_title.md @@ -0,0 +1 @@ +Explicitly version for php dependencies From d260e23787fb4942b151ed7516fc8a7c4207ba12 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 9 Jun 2020 19:33:15 +0200 Subject: [PATCH 2/7] Remove tmp hack to not push all apps --- autopatches/autopatch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/autopatches/autopatch.py b/autopatches/autopatch.py index 84662cff..1ace3b78 100755 --- a/autopatches/autopatch.py +++ b/autopatches/autopatch.py @@ -102,8 +102,6 @@ def push(patch): apps_to_push = [app for app in apps() if diff_not_empty(app) and app_is_on_github(app)] - apps_to_push = [app for app in apps() if app["id"] == "piwigo"] - with requests.Session() as s: s.headers.update({"Authorization": f"token {token}"}) for app in progressbar(apps_to_push, "Forking: ", 40): From 175fd4156ec0d27cba81afaadc2ecba4dbc9afc8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 10 Jun 2020 16:58:31 +0200 Subject: [PATCH 3/7] Cosmetics for diff --- autopatches/autopatch.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/autopatches/autopatch.py b/autopatches/autopatch.py index 1ace3b78..2013d731 100755 --- a/autopatches/autopatch.py +++ b/autopatches/autopatch.py @@ -5,11 +5,6 @@ import requests import os import subprocess -# ./autopatch.py --rebuild-cache -# ./autopatch.py --apply my_patch -# ./autopatch.py --diff -# ./autopatch.py --push my_patch - catalog = requests.get("https://raw.githubusercontent.com/YunoHost/apps/master/apps.json").json() my_env = os.environ.copy() @@ -83,10 +78,14 @@ def apply(patch): def diff(): for app in apps(): - print(app["id"]) - print("----------------------------") folder = os.path.join(".apps_cache", app["id"]) - os.system(f"cd {folder} && git diff && echo $?") + if bool(subprocess.check_output(f"cd {folder} && git diff", shell=True).strip().decode("utf-8")): + print("\n\n\n") + print("=================================") + print("Changes in : " + app["id"]) + print("=================================") + print("\n") + os.system(f"cd {folder} && git diff") def push(patch): From 77b148f67d844266c27142e69182f8ac87a93417 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Feb 2021 21:55:01 +0100 Subject: [PATCH 4/7] Add autopatch for new permission system --- autopatches/autopatch.py | 4 +- .../patches/new-permission-system/patch.sh | 66 +++++++++++++++++++ .../patches/new-permission-system/pr_body.md | 10 +++ .../patches/new-permission-system/pr_title.md | 1 + 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 autopatches/patches/new-permission-system/patch.sh create mode 100644 autopatches/patches/new-permission-system/pr_body.md create mode 100644 autopatches/patches/new-permission-system/pr_title.md diff --git a/autopatches/autopatch.py b/autopatches/autopatch.py index 2013d731..12235525 100755 --- a/autopatches/autopatch.py +++ b/autopatches/autopatch.py @@ -59,7 +59,7 @@ def build_cache(): for app in progressbar(apps(), "Git cloning: ", 40): folder = os.path.join(".apps_cache", app["id"]) - reponame = app["url"].rsplit("/", 1) + reponame = app["url"].rsplit("/", 1)[-1] git(f"clone --quiet --depth 1 --single-branch {app['url']} {folder}") git(f"remote add fork https://{login}:{token}@github.com/{login}/{reponame}", in_folder=folder) @@ -85,7 +85,7 @@ def diff(): print("Changes in : " + app["id"]) print("=================================") print("\n") - os.system(f"cd {folder} && git diff") + os.system(f"cd {folder} && git --no-pager diff") def push(patch): diff --git a/autopatches/patches/new-permission-system/patch.sh b/autopatches/patches/new-permission-system/patch.sh new file mode 100644 index 00000000..f372a44f --- /dev/null +++ b/autopatches/patches/new-permission-system/patch.sh @@ -0,0 +1,66 @@ + +cd scripts/ + +if grep -q 'ynh_legacy_permissions' upgrade || grep -q 'ynh_permission_' install +then + # App already using the new permission system - not patching anything + exit 0 +fi + +if ! grep -q "protected_\|skipped_" install +then + # App doesn't has any (un)protected / skipped setting ? + # Probably not a webapp or permission ain't relevant for it ? + exit 0 +fi + +CONFIGURE_PERMISSION_DURING_INSTALL=' +# Make app public if necessary +if [ \"\$is_public\" -eq 1 ] +then + ynh_permission_update --permission=\"main\" --add=\"visitors\" +fi +' + +MIGRATE_LEGACY_PERMISSIONS=' +#================================================= +# Migrate legacy permissions to new system +#================================================= +if ynh_legacy_permissions_exists +then + ynh_legacy_permissions_delete_all + + ynh_app_setting_delete --app=\$app --key=is_public +fi' + +for SCRIPT in "remove upgrade backup restore change_url" +do + [[ -e $SCRIPT ]] || continue + + perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i $SCRIPT + perl -p0e 's@\s*if.*-z.*is_public.*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@\s*if.*is_public.*(-eq|=).*(.|\n)*?fi\s@\n@g' -i $SCRIPT + perl -p0e 's@is_public=.*\n@@g' -i $SCRIPT + perl -p0e 's@ynh_app_setting_.*is_public.*@@g' -i $SCRIPT + perl -p0e 's@.*# Make app .*@@g' -i $SCRIPT + perl -p0e 's@.*# Fix is_public as a boolean.*@@g' -i $SCRIPT + perl -p0e 's@.*# If app is public.*@@g' -i $SCRIPT + perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i $SCRIPT + perl -p0e 's@.*ynh_script_progression.*SSOwat.*@@g' -i $SCRIPT + perl -p0e 's@#=*\s#.*SETUP SSOWAT.*\s#=*\s@@g' -i $SCRIPT +done + + +perl -p0e 's@.*domain_regex.*@@g' -i install +perl -p0e 's@.*# If app is public.*@@g' -i install +perl -p0e 's@.*# Make app .*@@g' -i install +perl -p0e 's@.*# .*allow.*credentials.*anyway.*@@g' -i install +perl -p0e "s@if.*is_public.*(-eq|=)(.|\n){0,100}setting(.|\n)*?fi\n@$CONFIGURE_PERMISSION_DURING_INSTALL@g" -i install +perl -p0e 's@.*ynh_app_setting_.*is_public.*\s@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*protected_.*@@g' -i install +perl -p0e 's@.*ynh_app_setting_.*skipped_.*@@g' -i install + +grep -q 'is_public=' install || perl -p0e 's@(.*Configuring SSOwat.*)@\1\nynh_permission_update --permission=\"main\" --add=\"visitors\"@g' -i install + +perl -p0e "s@ynh_abort_if_errors@ynh_abort_if_errors\n$MIGRATE_LEGACY_PERMISSIONS@g" -i upgrade diff --git a/autopatches/patches/new-permission-system/pr_body.md b/autopatches/patches/new-permission-system/pr_body.md new file mode 100644 index 00000000..115e26c0 --- /dev/null +++ b/autopatches/patches/new-permission-system/pr_body.md @@ -0,0 +1,10 @@ + +NB. : this is an ***automated*** attempt to migrate the app to the new permission system + +You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle: +- situations which are more complex than "if is_public is true, allow visitors" +- situations where the app needs to be temporarily public (then possible private) during initial configuration +- apps using non-standard syntax +- other specific use cases + +***PLEASE*** carefully review, test and amend the proposed changes if you find that the autopatch did not do a proper job. diff --git a/autopatches/patches/new-permission-system/pr_title.md b/autopatches/patches/new-permission-system/pr_title.md new file mode 100644 index 00000000..0f9554c9 --- /dev/null +++ b/autopatches/patches/new-permission-system/pr_title.md @@ -0,0 +1 @@ +Autopatch to migrate to new permission system From f384b00b112d3664133b1f776e18e33c55908749 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Feb 2021 22:01:29 +0100 Subject: [PATCH 5/7] Add note about extra permissions --- autopatches/patches/new-permission-system/pr_body.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autopatches/patches/new-permission-system/pr_body.md b/autopatches/patches/new-permission-system/pr_body.md index 115e26c0..07f4e855 100644 --- a/autopatches/patches/new-permission-system/pr_body.md +++ b/autopatches/patches/new-permission-system/pr_body.md @@ -4,6 +4,7 @@ NB. : this is an ***automated*** attempt to migrate the app to the new permissio You should ***not*** blindly trust the proposed changes. In particular, the auto-patch will not handle: - situations which are more complex than "if is_public is true, allow visitors" - situations where the app needs to be temporarily public (then possible private) during initial configuration +- apps that need to define extra permission for specific section of the app (such as admin interface) - apps using non-standard syntax - other specific use cases From f6531db64e1936b72b3cf3beccbf97bace5cb532 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 21 Feb 2021 22:02:28 +0100 Subject: [PATCH 6/7] Drop the old php patch --- .../patches/explicit-php-version-in-deps/patch.sh | 12 ------------ .../patches/explicit-php-version-in-deps/pr_body.md | 1 - .../patches/explicit-php-version-in-deps/pr_title.md | 1 - 3 files changed, 14 deletions(-) delete mode 100644 autopatches/patches/explicit-php-version-in-deps/patch.sh delete mode 100644 autopatches/patches/explicit-php-version-in-deps/pr_body.md delete mode 100644 autopatches/patches/explicit-php-version-in-deps/pr_title.md diff --git a/autopatches/patches/explicit-php-version-in-deps/patch.sh b/autopatches/patches/explicit-php-version-in-deps/patch.sh deleted file mode 100644 index cddbb44a..00000000 --- a/autopatches/patches/explicit-php-version-in-deps/patch.sh +++ /dev/null @@ -1,12 +0,0 @@ -PHP_DEPS="php-bcmath php-cli php-curl php-dev php-gd php-gmp php-imap php-intl php-json php-ldap php-mbstring php-mysql php-soap php-sqlite3 php-tidy php-xml php-xmlrpc php-zip php-dom php-opcache php-xsl php-apcu php-geoip php-imagick php-memcached php-redis php-ssh2 php-common" - -grep -q -nr "php-" scripts/* || exit 0 - -for DEP in $PHP_DEPS -do - NEWDEP=$(echo $DEP | sed 's/php-/php7.0-/g') - [ ! -e ./scripts/_common.sh ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/_common.sh - [ ! -e ./scripts/install ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/install - [ ! -e ./scripts/upgrade ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/upgrade - [ ! -e ./scripts/restore ] || sed "/^\s*#/!s/$DEP/$NEWDEP/g" -i ./scripts/restore -done diff --git a/autopatches/patches/explicit-php-version-in-deps/pr_body.md b/autopatches/patches/explicit-php-version-in-deps/pr_body.md deleted file mode 100644 index 2d8caed3..00000000 --- a/autopatches/patches/explicit-php-version-in-deps/pr_body.md +++ /dev/null @@ -1 +0,0 @@ -(This a test for now, please ignore, sorry for the noise) diff --git a/autopatches/patches/explicit-php-version-in-deps/pr_title.md b/autopatches/patches/explicit-php-version-in-deps/pr_title.md deleted file mode 100644 index 8b7f6417..00000000 --- a/autopatches/patches/explicit-php-version-in-deps/pr_title.md +++ /dev/null @@ -1 +0,0 @@ -Explicitly version for php dependencies From c94a3c00081600cb441bbfe019a61ea4d5971ca5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 22 Feb 2021 18:58:25 +0100 Subject: [PATCH 7/7] Add patch for missing ynh_abort_if_errors in change_url scripts --- .../patches/missing-seteu-in-change_url/patch.sh | 10 ++++++++++ .../patches/missing-seteu-in-change_url/pr_body.md | 2 ++ .../patches/missing-seteu-in-change_url/pr_title.md | 1 + 3 files changed, 13 insertions(+) create mode 100644 autopatches/patches/missing-seteu-in-change_url/patch.sh create mode 100644 autopatches/patches/missing-seteu-in-change_url/pr_body.md create mode 100644 autopatches/patches/missing-seteu-in-change_url/pr_title.md diff --git a/autopatches/patches/missing-seteu-in-change_url/patch.sh b/autopatches/patches/missing-seteu-in-change_url/patch.sh new file mode 100644 index 00000000..f9301348 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/patch.sh @@ -0,0 +1,10 @@ + +cd scripts/ + +if [ ! -e change_url ] || grep -q 'ynh_abort_if_errors' change_url +then + # The app doesn't has any change url script or already has ynh_abort_if_error + exit 0 +fi + +sed 's@\(source /usr/share/yunohost/helpers\)@\1\nynh_abort_if_errors@g' -i change_url diff --git a/autopatches/patches/missing-seteu-in-change_url/pr_body.md b/autopatches/patches/missing-seteu-in-change_url/pr_body.md new file mode 100644 index 00000000..bb67bd05 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/pr_body.md @@ -0,0 +1,2 @@ + +This is an ***automated*** patch to fix the lack of `ynh_abort_if_errors` in change_url script diff --git a/autopatches/patches/missing-seteu-in-change_url/pr_title.md b/autopatches/patches/missing-seteu-in-change_url/pr_title.md new file mode 100644 index 00000000..efd0e735 --- /dev/null +++ b/autopatches/patches/missing-seteu-in-change_url/pr_title.md @@ -0,0 +1 @@ +Missing ynh_abort_if_errors in change_url scripts