From 78df7de37f3ae71b29834bfa5a98c8fb67562c61 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 01:02:36 +0200 Subject: [PATCH 01/19] script to mirror the packages catalog on our forgejo --- tools/packages_repo_clone/.gitignore | 3 + tools/packages_repo_clone/clone_packages.py | 73 +++++++++++++++++++++ tools/packages_repo_clone/config.py.example | 3 + tools/packages_repo_clone/requirements.txt | 2 + 4 files changed, 81 insertions(+) create mode 100644 tools/packages_repo_clone/.gitignore create mode 100644 tools/packages_repo_clone/clone_packages.py create mode 100644 tools/packages_repo_clone/config.py.example create mode 100644 tools/packages_repo_clone/requirements.txt diff --git a/tools/packages_repo_clone/.gitignore b/tools/packages_repo_clone/.gitignore new file mode 100644 index 0000000..dd3dcf2 --- /dev/null +++ b/tools/packages_repo_clone/.gitignore @@ -0,0 +1,3 @@ +venv +__pycache__ +config.py \ No newline at end of file diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py new file mode 100644 index 0000000..caf572e --- /dev/null +++ b/tools/packages_repo_clone/clone_packages.py @@ -0,0 +1,73 @@ +from pathlib import Path +import requests +import toml +import time +import os +import config + + +APPS_REPO_ROOT = config.APPS_REPO_ROOT +GITHUB_TOKEN = config.GITHUB_TOKEN +FORGEJO_TOKEN = config.FORGEJO_TOKEN + +catalog = toml.load(open(os.path.join(APPS_REPO_ROOT, "apps.toml"), encoding="utf-8")) + + +def generate_catalog_repo_list(): + apps_repos = [] + for app in catalog: + url = catalog.get(app)["url"] + name = url.split("/")[-1] # get the last part of the URL as the repo name + apps_repos.append([name, url]) + return apps_repos + + +def generate_actual_repo_list(): + data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] + existing_clones = [] + for repo in data: + existing_clones.append(repo["name"]) + return existing_clones + + +def generate_mirrors(): + for app in generate_catalog_repo_list(): + repo_name = app[0] + repo_url = app[1] + if app[0] not in generate_actual_repo_list(): + print(f"A mirror for '{repo_name}' must be created.") + api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'} + create_mirror_data = { + "clone_addr": repo_url, + "auth_token": GITHUB_TOKEN, + "mirror": True, + "repo_name": repo_name, + "repo_owner": "YunoHost-Apps", + "service": "github" + } + create_mirror = requests.post("https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=create_mirror_data) + if create_mirror.status_code != 201: + if create_mirror.status_code == 409: + print(f"A repo named '{repo_name}' is already existing.") + if create_mirror.status_code == 422: + print(f"We're rate limited. Waiting for 1 minute before continuing.") + time.sleep(60) + else: + raise Exception('Request failed:', create_mirror.status_code, create_mirror.text) + else: + # configuring properly the new repository + settings_mirror_data = { + "has_packages": False, + "has_projects": False, + "has_releases": False, + "has_wiki": False + } + settings_mirror = requests.patch(f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=settings_mirror_data) + if settings_mirror.status_code != 200: + raise Exception('Request failed:', settings_mirror.text) + else: + print("Repository cloned and configured.") + time.sleep(5) # Sleeping for 5 seconds to cooldown the API + + +generate_mirrors() diff --git a/tools/packages_repo_clone/config.py.example b/tools/packages_repo_clone/config.py.example new file mode 100644 index 0000000..c13933c --- /dev/null +++ b/tools/packages_repo_clone/config.py.example @@ -0,0 +1,3 @@ +APPS_REPO_ROOT = "/path/to/the/catalog/repo" +GITHUB_TOKEN = "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +FORGEJO_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" \ No newline at end of file diff --git a/tools/packages_repo_clone/requirements.txt b/tools/packages_repo_clone/requirements.txt new file mode 100644 index 0000000..979a82a --- /dev/null +++ b/tools/packages_repo_clone/requirements.txt @@ -0,0 +1,2 @@ +requests +toml From f0931e5f8c39b6dfd7dd7a86e5627008d7cc21f7 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 01:24:29 +0200 Subject: [PATCH 02/19] smol improvements --- tools/packages_repo_clone/clone_packages.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index caf572e..2cc652b 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -14,6 +14,8 @@ catalog = toml.load(open(os.path.join(APPS_REPO_ROOT, "apps.toml"), encoding="ut def generate_catalog_repo_list(): + # list the apps in the catalog + apps_repos = [] for app in catalog: url = catalog.get(app)["url"] @@ -22,7 +24,9 @@ def generate_catalog_repo_list(): return apps_repos -def generate_actual_repo_list(): +def generate_mirror_list(): + # list the existing mirrors on our forgejo + data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] existing_clones = [] for repo in data: @@ -31,11 +35,17 @@ def generate_actual_repo_list(): def generate_mirrors(): - for app in generate_catalog_repo_list(): + + app_catalog = generate_catalog_repo_list() + mirror_list = generate_mirror_list() + + for app in app_catalog: repo_name = app[0] repo_url = app[1] - if app[0] not in generate_actual_repo_list(): + + if app[0] not in mirror_list: print(f"A mirror for '{repo_name}' must be created.") + api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'} create_mirror_data = { "clone_addr": repo_url, @@ -54,6 +64,7 @@ def generate_mirrors(): time.sleep(60) else: raise Exception('Request failed:', create_mirror.status_code, create_mirror.text) + else: # configuring properly the new repository settings_mirror_data = { From efb23356bd55282f3bec6e59208f4b2e3add8dc5 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 01:48:04 +0200 Subject: [PATCH 03/19] ignore packages we don't host --- tools/packages_repo_clone/clone_packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 2cc652b..4febe46 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -43,6 +43,9 @@ def generate_mirrors(): repo_name = app[0] repo_url = app[1] + if "https://github.com/YunoHost-Apps/" not in repo_url: + continue + if app[0] not in mirror_list: print(f"A mirror for '{repo_name}' must be created.") From e546df1abcaded760ab4fa3baaeaf8e6d6e51be3 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:25:12 +0200 Subject: [PATCH 04/19] refactor(clone_packages): remove useless import --- tools/packages_repo_clone/clone_packages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 4febe46..32aa151 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -1,4 +1,3 @@ -from pathlib import Path import requests import toml import time From 776e2983a9dcd85b1613a8c1601cc061c1e21b31 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:25:23 +0200 Subject: [PATCH 05/19] refactor(clone_packages): sort imports --- tools/packages_repo_clone/clone_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 32aa151..2d2bd7b 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -1,8 +1,8 @@ -import requests +import os import toml import time -import os import config +import requests APPS_REPO_ROOT = config.APPS_REPO_ROOT From 5f1d76a0f0c4245d7cc2a42b2a0cba09f0c16f5e Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:26:11 +0200 Subject: [PATCH 06/19] feat(clone_packages): cache actual repo list --- tools/packages_repo_clone/clone_packages.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 2d2bd7b..6205f73 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -14,7 +14,7 @@ catalog = toml.load(open(os.path.join(APPS_REPO_ROOT, "apps.toml"), encoding="ut def generate_catalog_repo_list(): # list the apps in the catalog - + apps_repos = [] for app in catalog: url = catalog.get(app)["url"] @@ -25,7 +25,7 @@ def generate_catalog_repo_list(): def generate_mirror_list(): # list the existing mirrors on our forgejo - + data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] existing_clones = [] for repo in data: @@ -34,20 +34,19 @@ def generate_mirror_list(): def generate_mirrors(): - app_catalog = generate_catalog_repo_list() mirror_list = generate_mirror_list() - + for app in app_catalog: repo_name = app[0] repo_url = app[1] - + if "https://github.com/YunoHost-Apps/" not in repo_url: continue - + if app[0] not in mirror_list: print(f"A mirror for '{repo_name}' must be created.") - + api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'} create_mirror_data = { "clone_addr": repo_url, @@ -66,7 +65,7 @@ def generate_mirrors(): time.sleep(60) else: raise Exception('Request failed:', create_mirror.status_code, create_mirror.text) - + else: # configuring properly the new repository settings_mirror_data = { From fb34339a66abe6328a2cce99acb339405575ec13 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:27:05 +0200 Subject: [PATCH 07/19] refactor(clone_packages): white lines to make code more readables --- tools/packages_repo_clone/clone_packages.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 6205f73..5de0394 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -16,20 +16,26 @@ def generate_catalog_repo_list(): # list the apps in the catalog apps_repos = [] + for app in catalog: url = catalog.get(app)["url"] name = url.split("/")[-1] # get the last part of the URL as the repo name + apps_repos.append([name, url]) + return apps_repos def generate_mirror_list(): # list the existing mirrors on our forgejo - data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] existing_clones = [] + + data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] + for repo in data: existing_clones.append(repo["name"]) + return existing_clones @@ -48,6 +54,7 @@ def generate_mirrors(): print(f"A mirror for '{repo_name}' must be created.") api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'} + create_mirror_data = { "clone_addr": repo_url, "auth_token": GITHUB_TOKEN, @@ -56,13 +63,17 @@ def generate_mirrors(): "repo_owner": "YunoHost-Apps", "service": "github" } + create_mirror = requests.post("https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=create_mirror_data) + if create_mirror.status_code != 201: if create_mirror.status_code == 409: print(f"A repo named '{repo_name}' is already existing.") + if create_mirror.status_code == 422: print(f"We're rate limited. Waiting for 1 minute before continuing.") time.sleep(60) + else: raise Exception('Request failed:', create_mirror.status_code, create_mirror.text) @@ -74,7 +85,9 @@ def generate_mirrors(): "has_releases": False, "has_wiki": False } + settings_mirror = requests.patch(f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=settings_mirror_data) + if settings_mirror.status_code != 200: raise Exception('Request failed:', settings_mirror.text) else: From 8786533ec6383c2cf936456297b17c4ae21defe3 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:28:00 +0200 Subject: [PATCH 08/19] style(clone_packages): black code --- tools/packages_repo_clone/clone_packages.py | 41 +++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 5de0394..d4be6bb 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -31,7 +31,9 @@ def generate_mirror_list(): existing_clones = [] - data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"] + data = requests.get( + "https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror" + ).json()["data"] for repo in data: existing_clones.append(repo["name"]) @@ -53,7 +55,10 @@ def generate_mirrors(): if app[0] not in mirror_list: print(f"A mirror for '{repo_name}' must be created.") - api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'} + api_header = { + "Content-type": "application/json", + "Authorization": "{FORGEJO_TOKEN}", + } create_mirror_data = { "clone_addr": repo_url, @@ -61,21 +66,30 @@ def generate_mirrors(): "mirror": True, "repo_name": repo_name, "repo_owner": "YunoHost-Apps", - "service": "github" - } + "service": "github", + } - create_mirror = requests.post("https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=create_mirror_data) + create_mirror = requests.post( + "https://git.yunohost.org/api/v1/repos/migrate", + headers=api_header, + params=f"access_token={FORGEJO_TOKEN}", + json=create_mirror_data, + ) if create_mirror.status_code != 201: if create_mirror.status_code == 409: print(f"A repo named '{repo_name}' is already existing.") if create_mirror.status_code == 422: - print(f"We're rate limited. Waiting for 1 minute before continuing.") + print( + f"We're rate limited. Waiting for 1 minute before continuing." + ) time.sleep(60) else: - raise Exception('Request failed:', create_mirror.status_code, create_mirror.text) + raise Exception( + "Request failed:", create_mirror.status_code, create_mirror.text + ) else: # configuring properly the new repository @@ -83,13 +97,18 @@ def generate_mirrors(): "has_packages": False, "has_projects": False, "has_releases": False, - "has_wiki": False - } + "has_wiki": False, + } - settings_mirror = requests.patch(f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=settings_mirror_data) + settings_mirror = requests.patch( + f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", + headers=api_header, + params=f"access_token={FORGEJO_TOKEN}", + json=settings_mirror_data, + ) if settings_mirror.status_code != 200: - raise Exception('Request failed:', settings_mirror.text) + raise Exception("Request failed:", settings_mirror.text) else: print("Repository cloned and configured.") time.sleep(5) # Sleeping for 5 seconds to cooldown the API From 5a2c9ca8e7b08fff689c4f3b859b762bb625035c Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:31:23 +0200 Subject: [PATCH 09/19] refactor(clone_packages): reduce indentation for lisibility (and fix two bugs) --- tools/packages_repo_clone/clone_packages.py | 99 ++++++++++----------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index d4be6bb..16d2a58 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -55,63 +55,62 @@ def generate_mirrors(): if app[0] not in mirror_list: print(f"A mirror for '{repo_name}' must be created.") - api_header = { - "Content-type": "application/json", - "Authorization": "{FORGEJO_TOKEN}", - } + print(f"A mirror for '{repo_name}' must be created.") - create_mirror_data = { - "clone_addr": repo_url, - "auth_token": GITHUB_TOKEN, - "mirror": True, - "repo_name": repo_name, - "repo_owner": "YunoHost-Apps", - "service": "github", - } + api_header = { + "Content-type": "application/json", + "Authorization": "{FORGEJO_TOKEN}", + } - create_mirror = requests.post( - "https://git.yunohost.org/api/v1/repos/migrate", - headers=api_header, - params=f"access_token={FORGEJO_TOKEN}", - json=create_mirror_data, + create_mirror_data = { + "clone_addr": repo_url, + "auth_token": GITHUB_TOKEN, + "mirror": True, + "repo_name": repo_name, + "repo_owner": "YunoHost-Apps", + "service": "github", + } + + create_mirror = requests.post( + "https://git.yunohost.org/api/v1/repos/migrate", + headers=api_header, + params=f"access_token={FORGEJO_TOKEN}", + json=create_mirror_data, + ) + + if create_mirror.status_code == 409: + print(f"A repo named '{repo_name}' is already existing.") + continue + + if create_mirror.status_code == 422: + print("We're rate limited. Waiting for 1 minute before continuing.") + time.sleep(60) + + if create_mirror.status_code not in (201, 422): + raise Exception( + "Request failed:", create_mirror.status_code, create_mirror.text ) - if create_mirror.status_code != 201: - if create_mirror.status_code == 409: - print(f"A repo named '{repo_name}' is already existing.") + # configuring properly the new repository + settings_mirror_data = { + "has_packages": False, + "has_projects": False, + "has_releases": False, + "has_wiki": False, + } - if create_mirror.status_code == 422: - print( - f"We're rate limited. Waiting for 1 minute before continuing." - ) - time.sleep(60) + settings_mirror = requests.patch( + f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", + headers=api_header, + params=f"access_token={FORGEJO_TOKEN}", + json=settings_mirror_data, + ) - else: - raise Exception( - "Request failed:", create_mirror.status_code, create_mirror.text - ) + if settings_mirror.status_code != 200: + raise Exception("Request failed:", settings_mirror.text) - else: - # configuring properly the new repository - settings_mirror_data = { - "has_packages": False, - "has_projects": False, - "has_releases": False, - "has_wiki": False, - } - - settings_mirror = requests.patch( - f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", - headers=api_header, - params=f"access_token={FORGEJO_TOKEN}", - json=settings_mirror_data, - ) - - if settings_mirror.status_code != 200: - raise Exception("Request failed:", settings_mirror.text) - else: - print("Repository cloned and configured.") - time.sleep(5) # Sleeping for 5 seconds to cooldown the API + print("Repository cloned and configured.") + time.sleep(5) # Sleeping for 5 seconds to cooldown the API generate_mirrors() From cc3330a2788d8943034096acde03fba289e6565f Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:33:14 +0200 Subject: [PATCH 10/19] refactor(clone_packages): rename variable --- tools/packages_repo_clone/clone_packages.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 16d2a58..9758381 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -71,24 +71,24 @@ def generate_mirrors(): "service": "github", } - create_mirror = requests.post( + create_mirror_request = requests.post( "https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=create_mirror_data, ) - if create_mirror.status_code == 409: + if create_mirror_request.status_code == 409: print(f"A repo named '{repo_name}' is already existing.") continue - if create_mirror.status_code == 422: + if create_mirror_request.status_code == 422: print("We're rate limited. Waiting for 1 minute before continuing.") time.sleep(60) - if create_mirror.status_code not in (201, 422): + if create_mirror_request.status_code not in (201, 422): raise Exception( - "Request failed:", create_mirror.status_code, create_mirror.text + "Request failed:", create_mirror_request.status_code, create_mirror_request.text ) # configuring properly the new repository From d816bd62552bd017460c0fa6124458b32da1a669 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:34:07 +0200 Subject: [PATCH 11/19] refactor(clone_packages): rename variable --- tools/packages_repo_clone/clone_packages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 9758381..6135510 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -99,15 +99,15 @@ def generate_mirrors(): "has_wiki": False, } - settings_mirror = requests.patch( + settings_mirror_request = requests.patch( f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=settings_mirror_data, ) - if settings_mirror.status_code != 200: - raise Exception("Request failed:", settings_mirror.text) + if settings_mirror_request.status_code != 200: + raise Exception("Request failed:", settings_mirror_request.text) print("Repository cloned and configured.") time.sleep(5) # Sleeping for 5 seconds to cooldown the API From 8a71ebbd1e9e3d416c50416d248307ff9ccff431 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:47:48 +0200 Subject: [PATCH 12/19] feat(clone_packages): handle api rate limit --- tools/packages_repo_clone/clone_packages.py | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 6135510..80f1fbf 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -41,6 +41,20 @@ def generate_mirror_list(): return existing_clones +def request_handling_rate_limit(method, *args, **kwargs): + while True: + response = method(*args, **kwargs) + + # we are not reated limited + if response.status_code != 422: + break + + print("We're rate limited. Waiting for 1 minute before continuing.") + time.sleep(60) + + return response + + def generate_mirrors(): app_catalog = generate_catalog_repo_list() mirror_list = generate_mirror_list() @@ -71,7 +85,8 @@ def generate_mirrors(): "service": "github", } - create_mirror_request = requests.post( + create_mirror_request = request_handling_rate_limit( + requests.post, "https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", @@ -82,11 +97,7 @@ def generate_mirrors(): print(f"A repo named '{repo_name}' is already existing.") continue - if create_mirror_request.status_code == 422: - print("We're rate limited. Waiting for 1 minute before continuing.") - time.sleep(60) - - if create_mirror_request.status_code not in (201, 422): + if create_mirror_request.status_code != 201: raise Exception( "Request failed:", create_mirror_request.status_code, create_mirror_request.text ) @@ -99,7 +110,8 @@ def generate_mirrors(): "has_wiki": False, } - settings_mirror_request = requests.patch( + settings_mirror_request = request_handling_rate_limit( + requests.patch, f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", From 16dd17af59848a8a1a10db6796af833aff64811a Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 01:48:03 +0200 Subject: [PATCH 13/19] feat(clone_packages): ensure that all request have a timeout --- tools/packages_repo_clone/clone_packages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 80f1fbf..bd755ba 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -42,6 +42,9 @@ def generate_mirror_list(): def request_handling_rate_limit(method, *args, **kwargs): + if "timeout" not in kwargs: + kwargs["timeout"] = 10 + while True: response = method(*args, **kwargs) From ce66396fdc9beeb982ced24ce459d48ec2862049 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 21:45:43 +0200 Subject: [PATCH 14/19] fixes double print, adds comments --- tools/packages_repo_clone/clone_packages.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index bd755ba..277cfbd 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -48,7 +48,7 @@ def request_handling_rate_limit(method, *args, **kwargs): while True: response = method(*args, **kwargs) - # we are not reated limited + # we are not rate limited if response.status_code != 422: break @@ -66,11 +66,13 @@ def generate_mirrors(): repo_name = app[0] repo_url = app[1] - if "https://github.com/YunoHost-Apps/" not in repo_url: + if not repo_url.startswith("https://github.com/YunoHost-Apps/"): + # do not care about repos that are not in the YunoHost-Apps org continue - if app[0] not in mirror_list: - print(f"A mirror for '{repo_name}' must be created.") + if repo_name in mirror_list: + # the mirror is already existing + continue print(f"A mirror for '{repo_name}' must be created.") @@ -102,7 +104,9 @@ def generate_mirrors(): if create_mirror_request.status_code != 201: raise Exception( - "Request failed:", create_mirror_request.status_code, create_mirror_request.text + "Request failed:", + create_mirror_request.status_code, + create_mirror_request.text, ) # configuring properly the new repository From 145cd80f611ce128a20aeabda0c7dfeadeca7277 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 21:53:03 +0200 Subject: [PATCH 15/19] fix(clone_packages): add timeout on generate mirror list request --- tools/packages_repo_clone/clone_packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 277cfbd..4c442c0 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -32,7 +32,8 @@ def generate_mirror_list(): existing_clones = [] data = requests.get( - "https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror" + "https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror", + timeout=60, ).json()["data"] for repo in data: From d8c6d6f5ce2f2a34d832057efe847a9edbae6845 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 21:53:27 +0200 Subject: [PATCH 16/19] fix(clone_packages): 10 seconds default timeout was too short --- tools/packages_repo_clone/clone_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index 4c442c0..efe5612 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -44,7 +44,7 @@ def generate_mirror_list(): def request_handling_rate_limit(method, *args, **kwargs): if "timeout" not in kwargs: - kwargs["timeout"] = 10 + kwargs["timeout"] = 60 while True: response = method(*args, **kwargs) From 9e54b8ee32089c59d3185acbaa4b72e01c80f4a6 Mon Sep 17 00:00:00 2001 From: Laurent Peuch Date: Sun, 5 May 2024 22:04:07 +0200 Subject: [PATCH 17/19] fix(clone_packages): handling the pagination to get the repo list --- tools/packages_repo_clone/clone_packages.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/packages_repo_clone/clone_packages.py index efe5612..6b650c5 100644 --- a/tools/packages_repo_clone/clone_packages.py +++ b/tools/packages_repo_clone/clone_packages.py @@ -31,13 +31,21 @@ def generate_mirror_list(): existing_clones = [] - data = requests.get( - "https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror", - timeout=60, - ).json()["data"] + page = 1 + while True: + data = requests.get( + f"https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror&page={page}&limit=100", + timeout=60, + ).json()["data"] - for repo in data: - existing_clones.append(repo["name"]) + # once the data list is empty the whole available pages are consumed + if not data: + break + + for repo in data: + existing_clones.append(repo["name"]) + + page += 1 return existing_clones From 5ea2665b5f5d294f5c1b43bdc5f3e971af6bfb89 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 22:26:45 +0200 Subject: [PATCH 18/19] rename the folder --- tools/{packages_repo_clone => mirror_repos_forgejo}/.gitignore | 0 .../clone_packages.py | 0 .../config.py.example | 0 .../requirements.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tools/{packages_repo_clone => mirror_repos_forgejo}/.gitignore (100%) rename tools/{packages_repo_clone => mirror_repos_forgejo}/clone_packages.py (100%) rename tools/{packages_repo_clone => mirror_repos_forgejo}/config.py.example (100%) rename tools/{packages_repo_clone => mirror_repos_forgejo}/requirements.txt (100%) diff --git a/tools/packages_repo_clone/.gitignore b/tools/mirror_repos_forgejo/.gitignore similarity index 100% rename from tools/packages_repo_clone/.gitignore rename to tools/mirror_repos_forgejo/.gitignore diff --git a/tools/packages_repo_clone/clone_packages.py b/tools/mirror_repos_forgejo/clone_packages.py similarity index 100% rename from tools/packages_repo_clone/clone_packages.py rename to tools/mirror_repos_forgejo/clone_packages.py diff --git a/tools/packages_repo_clone/config.py.example b/tools/mirror_repos_forgejo/config.py.example similarity index 100% rename from tools/packages_repo_clone/config.py.example rename to tools/mirror_repos_forgejo/config.py.example diff --git a/tools/packages_repo_clone/requirements.txt b/tools/mirror_repos_forgejo/requirements.txt similarity index 100% rename from tools/packages_repo_clone/requirements.txt rename to tools/mirror_repos_forgejo/requirements.txt From 392942145d6b7f362631792e415430ba379614e5 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Sun, 5 May 2024 22:38:53 +0200 Subject: [PATCH 19/19] Print a 'Done.' when the script finished its work --- tools/mirror_repos_forgejo/clone_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/mirror_repos_forgejo/clone_packages.py b/tools/mirror_repos_forgejo/clone_packages.py index 6b650c5..538f1a0 100644 --- a/tools/mirror_repos_forgejo/clone_packages.py +++ b/tools/mirror_repos_forgejo/clone_packages.py @@ -140,5 +140,7 @@ def generate_mirrors(): print("Repository cloned and configured.") time.sleep(5) # Sleeping for 5 seconds to cooldown the API + print("Done.") + generate_mirrors()