From a652a3dae9f5d312fe6ce186b97b52e8f8364763 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 27 Mar 2018 13:50:29 +0200 Subject: [PATCH] [enh] Handle HEAD commits (#443) * Refactor list_builder requests stuff into functions * Fetch commit if revision is 'HEAD' --- list_builder.py | 150 +++++++++++++++++++++++++++--------------------- official.json | 2 +- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/list_builder.py b/list_builder.py index 47c4b389..8d24561e 100755 --- a/list_builder.py +++ b/list_builder.py @@ -62,6 +62,37 @@ def include_translations_in_manifest(app_name, manifest): return manifest +def get_json(url, verify=True): + + try: + # Retrieve and load manifest + if ".github" in url: + r = requests.get(url, verify=verify, auth=token) + else: + r = requests.get(url, verify=verify) + r.raise_for_status() + return r.json() + except requests.exceptions.RequestException as e: + print("-> Error: unable to request %s, %s" % (url, e)) + return None + except ValueError as e: + print("-> Error: unable to decode json from %s : %s" % (url, e)) + return None + +def get_zlib(url, verify=True): + + try: + # Retrieve last commit information + r = requests.get(obj_url, verify=verify) + r.raise_for_status() + return zlib.decompress(r.content).decode('utf-8').split('\x00') + except requests.exceptions.RequestException as e: + print("-> Error: unable to request %s, %s" % (obj_url, e)) + return None + except zlib.error as e: + print("-> Error: unable to decompress object from %s : %s" % (url, e)) + return None + # Main # Create argument parser @@ -106,14 +137,27 @@ else: # Loop through every apps result_dict = {} for app, info in apps_list.items(): + print("---") print("Processing '%s'..." % app) # Store usefull values + app_branch = info['branch'] app_url = info['url'] app_rev = info['revision'] app_state = info["state"] app_level = info.get("level") + github_repo = re_github_repo.match(app_url) + if github_repo and app_rev == "HEAD": + owner = github_repo.group('owner') + repo = github_repo.group('repo') + url = "https://api.github.com/repos/%s/%s/git/refs/heads/%s" % (owner, repo, app_branch) + ref_stuff = get_json(url) + if ref_stuff is None or not "object" in ref_stuff or not "sha" in ref_stuff["object"]: + print("-> Error, couldn't get the commit corresponding to HEAD ..") + continue + app_rev = ref_stuff["object"]["sha"] + previous_state = already_built_file.get(app, {}).get("state", {}) manifest = {} @@ -123,8 +167,11 @@ for app, info in apps_list.items(): previous_url = already_built_file.get(app, {}).get("git", {}).get("url") previous_level = already_built_file.get(app, {}).get("level") + print("Previous commit : %s" % previous_rev) + print("Current commit : %s" % app_rev) + if previous_rev == app_rev and previous_url == app_url: - print("%s[%s] is already up to date in target output, ignore" % (app, app_rev)) + print("Already up to date, ignoring") result_dict[app] = already_built_file[app] if previous_state != app_state: result_dict[app]["state"] = app_state @@ -138,8 +185,9 @@ for app, info in apps_list.items(): continue + print("Revision changed ! Updating...") + # Hosted on GitHub - github_repo = re_github_repo.match(app_url) if github_repo: owner = github_repo.group('owner') repo = github_repo.group('repo') @@ -147,87 +195,61 @@ for app, info in apps_list.items(): raw_url = 'https://raw.githubusercontent.com/%s/%s/%s/manifest.json' % ( owner, repo, app_rev ) - try: - # Retrieve and load manifest - r = requests.get(raw_url, auth=token) - r.raise_for_status() - manifest = r.json() - except requests.exceptions.RequestException as e: - print("-> Error: unable to request %s, %s" % (raw_url, e)) - continue - except ValueError as e: - print("-> Error: unable to decode manifest.json, %s" % e) + + manifest = get_json(raw_url) + if manifest is None: continue api_url = 'https://api.github.com/repos/%s/%s/commits/%s' % ( owner, repo, app_rev ) - try: - # Retrieve last commit information - r = requests.get(api_url, auth=token) - r.raise_for_status() - info2 = r.json() - except requests.exceptions.RequestException as e: - print("-> Error: unable to request %s, %s" % (api_url, e)) + + info2 = get_json(api_url) + if info2 is None: continue - except ValueError as e: - print("-> Error: unable to decode API response, %s" % e) - continue - else: - commit_date = parse(info2['commit']['author']['date']) - timestamp = int(time.mktime(commit_date.timetuple())) + + commit_date = parse(info2['commit']['author']['date']) + timestamp = int(time.mktime(commit_date.timetuple())) # Git repository with HTTP/HTTPS (Gogs, GitLab, ...) elif app_url.startswith('http') and app_url.endswith('.git'): + raw_url = '%s/raw/%s/manifest.json' % (app_url[:-4], app_rev) - try: - # Attempt to retrieve and load raw manifest - r = requests.get(raw_url, verify=False, auth=token) - r.raise_for_status() - manifest = r.json() - except requests.exceptions.RequestException as e: - print("-> Error: unable to request %s, %s" % (raw_url, e)) - continue - except ValueError as e: - print("-> Error: unable to decode manifest.json, %s" % e) + manifest = get_json(raw_url, verify=False) + if manifest is None: continue obj_url = '%s/objects/%s/%s' % ( app_url, app_rev[0:2], app_rev[2:] ) - try: - # Retrieve last commit information - r = requests.get(obj_url, verify=False) - r.raise_for_status() - commit = zlib.decompress(r.content).decode('utf-8').split('\x00')[1] - except requests.exceptions.RequestException as e: - print("-> Error: unable to request %s, %s" % (obj_url, e)) - continue - except zlib.error as e: - print("-> Error: unable to decompress commit object, %s" % e) + commit = get_zlib(obj_url, verify=False) + + if commit is None or len(commit) < 2: continue else: - # Extract author line and commit date - commit_author = re_commit_author.search(commit) - if not commit_author: - print("-> Error: author line in commit not found") - continue + commit = commit[1] - # Construct UTC timestamp - timestamp = int(commit_author.group('time')) - tz = commit_author.group('tz') - if len(tz) != 5: - print("-> Error: unexpected timezone length in commit") + # Extract author line and commit date + commit_author = re_commit_author.search(commit) + if not commit_author: + print("-> Error: author line in commit not found") + continue + + # Construct UTC timestamp + timestamp = int(commit_author.group('time')) + tz = commit_author.group('tz') + if len(tz) != 5: + print("-> Error: unexpected timezone length in commit") + continue + elif tz != '+0000': + tdelta = (int(tz[1:3]) * 3600) + (int(tz[3:5]) * 60) + if tz[0] == '+': + timestamp -= tdelta + elif tz[0] == '-': + timestamp += tdelta + else: + print("-> Error: unexpected timezone format in commit") continue - elif tz != '+0000': - tdelta = (int(tz[1:3]) * 3600) + (int(tz[3:5]) * 60) - if tz[0] == '+': - timestamp -= tdelta - elif tz[0] == '-': - timestamp += tdelta - else: - print("-> Error: unexpected timezone format in commit") - continue else: print("-> Error: unsupported VCS and/or protocol") continue diff --git a/official.json b/official.json index 18db7447..099e2809 100644 --- a/official.json +++ b/official.json @@ -30,7 +30,7 @@ "hextris": { "branch": "master", "level": 7, - "revision": "58326cc06c5ef8ad8094cd577170b0d1b096af5e", + "revision": "HEAD", "state": "validated", "url": "https://github.com/YunoHost-Apps/hextris_ynh" },