script to mirror the packages catalog on our forgejo

tools/packages_repo_clone/.gitignore

@@ -0,0 +1,3 @@
+venv
+__pycache__
+config.py

tools/packages_repo_clone/clone_packages.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+import requests
+import toml
+import time
+import os
+import config
+
+
+APPS_REPO_ROOT = config.APPS_REPO_ROOT
+GITHUB_TOKEN = config.GITHUB_TOKEN
+FORGEJO_TOKEN = config.FORGEJO_TOKEN
+
+catalog = toml.load(open(os.path.join(APPS_REPO_ROOT, "apps.toml"), encoding="utf-8"))
+
+
+def generate_catalog_repo_list():
+    apps_repos = []
+    for app in catalog:
+        url = catalog.get(app)["url"]
+        name = url.split("/")[-1]  # get the last part of the URL as the repo name
+        apps_repos.append([name, url])
+    return apps_repos
+
+
+def generate_actual_repo_list():
+    data = requests.get("https://git.yunohost.org/api/v1/repos/search?topic=false&includeDesc=false&priority_owner_id=17&mode=mirror").json()["data"]
+    existing_clones = []
+    for repo in data:
+        existing_clones.append(repo["name"])
+    return existing_clones
+
+
+def generate_mirrors():
+    for app in generate_catalog_repo_list():
+        repo_name = app[0]
+        repo_url = app[1]
+        if app[0] not in generate_actual_repo_list():
+            print(f"A mirror for '{repo_name}' must be created.")
+            api_header = {'Content-type': 'application/json', 'Authorization': '{FORGEJO_TOKEN}'}
+            create_mirror_data = {
+                "clone_addr": repo_url,
+                "auth_token": GITHUB_TOKEN,
+                "mirror": True,
+                "repo_name": repo_name,
+                "repo_owner": "YunoHost-Apps",
+                "service": "github"
+                }
+            create_mirror = requests.post("https://git.yunohost.org/api/v1/repos/migrate", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=create_mirror_data)
+            if create_mirror.status_code != 201:
+                if create_mirror.status_code == 409:
+                    print(f"A repo named '{repo_name}' is already existing.")
+                    if create_mirror.status_code == 422:
+                        print(f"We're rate limited. Waiting for 1 minute before continuing.")
+                        time.sleep(60)
+                else:
+                    raise Exception('Request failed:', create_mirror.status_code, create_mirror.text)
+            else:
+                # configuring properly the new repository
+                settings_mirror_data = {
+                    "has_packages": False,
+                    "has_projects": False,
+                    "has_releases": False,
+                    "has_wiki": False
+                    }
+                settings_mirror = requests.patch(f"https://git.yunohost.org/api/v1/repos/YunoHost-Apps/{repo_name}", headers=api_header, params=f"access_token={FORGEJO_TOKEN}", json=settings_mirror_data)
+                if settings_mirror.status_code != 200:
+                    raise Exception('Request failed:', settings_mirror.text)
+                else:
+                    print("Repository cloned and configured.")
+                    time.sleep(5)  # Sleeping for 5 seconds to cooldown the API
+
+
+generate_mirrors()

tools/packages_repo_clone/config.py.example

@@ -0,0 +1,3 @@
+APPS_REPO_ROOT = "/path/to/the/catalog/repo"
+GITHUB_TOKEN = "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+FORGEJO_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

tools/packages_repo_clone/requirements.txt

@@ -0,0 +1,2 @@
+requests
+toml