Merge branch 'master' of https://github.com/YunoHost/apps

list_builder.py

@@ -62,6 +62,37 @@ def include_translations_in_manifest(app_name, manifest):
     return manifest
 
 
+def get_json(url, verify=True):
+
+    try:
+        # Retrieve and load manifest
+        if ".github" in url:
+            r = requests.get(url, verify=verify, auth=token)
+        else:
+            r = requests.get(url, verify=verify)
+        r.raise_for_status()
+        return r.json()
+    except requests.exceptions.RequestException as e:
+        print("-> Error: unable to request %s, %s" % (url, e))
+        return None
+    except ValueError as e:
+        print("-> Error: unable to decode json from %s : %s" % (url, e))
+        return None
+
+def get_zlib(url, verify=True):
+
+    try:
+        # Retrieve last commit information
+        r = requests.get(obj_url, verify=verify)
+        r.raise_for_status()
+        return zlib.decompress(r.content).decode('utf-8').split('\x00')
+    except requests.exceptions.RequestException as e:
+        print("-> Error: unable to request %s, %s" % (obj_url, e))
+        return None
+    except zlib.error as e:
+        print("-> Error: unable to decompress object from %s : %s" % (url, e))
+        return None
+
 # Main
 
 # Create argument parser
@@ -106,14 +137,27 @@ else:
 # Loop through every apps
 result_dict = {}
 for app, info in apps_list.items():
+    print("---")
     print("Processing '%s'..." % app)
 
     # Store usefull values
+    app_branch = info['branch']
     app_url = info['url']
     app_rev = info['revision']
     app_state = info["state"]
     app_level = info.get("level")
 
+    github_repo = re_github_repo.match(app_url)
+    if github_repo and app_rev == "HEAD":
+        owner = github_repo.group('owner')
+        repo = github_repo.group('repo')
+        url = "https://api.github.com/repos/%s/%s/git/refs/heads/%s" % (owner, repo, app_branch)
+        ref_stuff = get_json(url)
+        if ref_stuff is None or not "object" in ref_stuff or not "sha" in ref_stuff["object"]:
+            print("-> Error, couldn't get the commit corresponding to HEAD ..")
+            continue
+        app_rev =  ref_stuff["object"]["sha"]
+
     previous_state = already_built_file.get(app, {}).get("state", {})
 
     manifest = {}
@@ -123,8 +167,11 @@ for app, info in apps_list.items():
     previous_url = already_built_file.get(app, {}).get("git", {}).get("url")
     previous_level = already_built_file.get(app, {}).get("level")
 
+    print("Previous commit : %s" % previous_rev)
+    print("Current commit : %s" % app_rev)
+
     if previous_rev == app_rev and previous_url == app_url:
-        print("%s[%s] is already up to date in target output, ignore" % (app, app_rev))
+        print("Already up to date, ignoring")
         result_dict[app] = already_built_file[app]
         if previous_state != app_state:
             result_dict[app]["state"] = app_state
@@ -138,8 +185,9 @@ for app, info in apps_list.items():
 
         continue
 
+    print("Revision changed ! Updating...")
+
     # Hosted on GitHub
-    github_repo = re_github_repo.match(app_url)
     if github_repo:
         owner = github_repo.group('owner')
         repo = github_repo.group('repo')
@@ -147,66 +195,40 @@ for app, info in apps_list.items():
         raw_url = 'https://raw.githubusercontent.com/%s/%s/%s/manifest.json' % (
             owner, repo, app_rev
         )
-        try:
+
-            # Retrieve and load manifest
+        manifest = get_json(raw_url)
-            r = requests.get(raw_url, auth=token)
+        if manifest is None:
-            r.raise_for_status()
-            manifest = r.json()
-        except requests.exceptions.RequestException as e:
-            print("-> Error: unable to request %s, %s" % (raw_url, e))
-            continue
-        except ValueError as e:
-            print("-> Error: unable to decode manifest.json, %s" % e)
             continue
 
         api_url = 'https://api.github.com/repos/%s/%s/commits/%s' % (
             owner, repo, app_rev
         )
-        try:
+
-            # Retrieve last commit information
+        info2 = get_json(api_url)
-            r = requests.get(api_url, auth=token)
+        if info2 is None:
-            r.raise_for_status()
-            info2 = r.json()
-        except requests.exceptions.RequestException as e:
-            print("-> Error: unable to request %s, %s" % (api_url, e))
             continue
-        except ValueError as e:
+
-            print("-> Error: unable to decode API response, %s" % e)
-            continue
-        else:
         commit_date = parse(info2['commit']['author']['date'])
         timestamp = int(time.mktime(commit_date.timetuple()))
 
     # Git repository with HTTP/HTTPS (Gogs, GitLab, ...)
     elif app_url.startswith('http') and app_url.endswith('.git'):
+
         raw_url = '%s/raw/%s/manifest.json' % (app_url[:-4], app_rev)
-        try:
+        manifest = get_json(raw_url, verify=False)
-            # Attempt to retrieve and load raw manifest
+        if manifest is None:
-            r = requests.get(raw_url, verify=False, auth=token)
-            r.raise_for_status()
-            manifest = r.json()
-        except requests.exceptions.RequestException as e:
-            print("-> Error: unable to request %s, %s" % (raw_url, e))
-            continue
-        except ValueError as e:
-            print("-> Error: unable to decode manifest.json, %s" % e)
             continue
 
         obj_url = '%s/objects/%s/%s' % (
             app_url, app_rev[0:2], app_rev[2:]
         )
-        try:
+        commit = get_zlib(obj_url, verify=False)
-            # Retrieve last commit information
+
-            r = requests.get(obj_url, verify=False)
+        if commit is None or len(commit) < 2:
-            r.raise_for_status()
-            commit = zlib.decompress(r.content).decode('utf-8').split('\x00')[1]
-        except requests.exceptions.RequestException as e:
-            print("-> Error: unable to request %s, %s" % (obj_url, e))
-            continue
-        except zlib.error as e:
-            print("-> Error: unable to decompress commit object, %s" % e)
             continue
         else:
+            commit = commit[1]
+
         # Extract author line and commit date
         commit_author = re_commit_author.search(commit)
         if not commit_author:

official.json

@@ -30,7 +30,7 @@
     "hextris": {
         "branch": "master",
         "level": 7,
-        "revision": "58326cc06c5ef8ad8094cd577170b0d1b096af5e",
+        "revision": "HEAD",
         "state": "validated",
         "url": "https://github.com/YunoHost-Apps/hextris_ynh"
     },