mirror of
https://github.com/YunoHost/yunodevtools.git
synced 2024-09-03 20:16:19 +02:00
black on list_builder.py
This commit is contained in:
Alexandre Aubin
parent
9db6bde4b0
commit
fe52e90610
1 changed files with 163 additions and 56 deletions
205
list_builder.py
205
list_builder.py
|
|
@ -11,7 +11,9 @@ import time
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|
||||||
catalog = json.load(open("apps.json"))
|
catalog = json.load(open("apps.json"))
|
||||||
catalog = {app: infos for app, infos in catalog.items() if infos.get('state') != 'notworking'}
|
catalog = {
|
||||||
|
app: infos for app, infos in catalog.items() if infos.get("state") != "notworking"
|
||||||
|
}
|
||||||
|
|
||||||
my_env = os.environ.copy()
|
my_env = os.environ.copy()
|
||||||
my_env["GIT_TERMINAL_PROMPT"] = "0"
|
my_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||||
|
|
@ -19,31 +21,39 @@ my_env["GIT_TERMINAL_PROMPT"] = "0"
|
||||||
os.makedirs(".apps_cache", exist_ok=True)
|
os.makedirs(".apps_cache", exist_ok=True)
|
||||||
os.makedirs("builds/", exist_ok=True)
|
os.makedirs("builds/", exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
def error(msg):
|
def error(msg):
|
||||||
msg = "[Applist builder error] " + msg
|
msg = "[Applist builder error] " + msg
|
||||||
if os.path.exists("/usr/bin/sendxmpppy"):
|
if os.path.exists("/usr/bin/sendxmpppy"):
|
||||||
subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, 'wb'))
|
subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, "wb"))
|
||||||
print(msg + "\n")
|
print(msg + "\n")
|
||||||
|
|
||||||
|
|
||||||
# Progress bar helper, stolen from https://stackoverflow.com/a/34482761
|
# Progress bar helper, stolen from https://stackoverflow.com/a/34482761
|
||||||
def progressbar(it, prefix="", size=60, file=sys.stdout):
|
def progressbar(it, prefix="", size=60, file=sys.stdout):
|
||||||
count = len(it)
|
count = len(it)
|
||||||
|
|
||||||
def show(j, name=""):
|
def show(j, name=""):
|
||||||
name += " "
|
name += " "
|
||||||
x = int(size*j/count)
|
x = int(size * j / count)
|
||||||
file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name))
|
file.write(
|
||||||
|
"%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name)
|
||||||
|
)
|
||||||
file.flush()
|
file.flush()
|
||||||
|
|
||||||
show(0)
|
show(0)
|
||||||
for i, item in enumerate(it):
|
for i, item in enumerate(it):
|
||||||
yield item
|
yield item
|
||||||
show(i+1, item[0])
|
show(i + 1, item[0])
|
||||||
file.write("\n")
|
file.write("\n")
|
||||||
file.flush()
|
file.flush()
|
||||||
|
|
||||||
|
|
||||||
###################################
|
###################################
|
||||||
# App git clones cache management #
|
# App git clones cache management #
|
||||||
###################################
|
###################################
|
||||||
|
|
||||||
|
|
||||||
def app_cache_folder(app):
|
def app_cache_folder(app):
|
||||||
return os.path.join(".apps_cache", app)
|
return os.path.join(".apps_cache", app)
|
||||||
|
|
||||||
|
|
@ -81,11 +91,13 @@ def init_cache(app, infos):
|
||||||
else:
|
else:
|
||||||
depth = 40
|
depth = 40
|
||||||
|
|
||||||
git("clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format(
|
git(
|
||||||
|
"clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format(
|
||||||
depth=depth,
|
depth=depth,
|
||||||
url=infos["url"],
|
url=infos["url"],
|
||||||
branch=infos.get("branch", "master"),
|
branch=infos.get("branch", "master"),
|
||||||
folder=app_cache_folder(app))
|
folder=app_cache_folder(app),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -102,15 +114,23 @@ def refresh_cache(app, infos):
|
||||||
git("remote set-url origin " + infos["url"], in_folder=app_cache_folder(app))
|
git("remote set-url origin " + infos["url"], in_folder=app_cache_folder(app))
|
||||||
# With git >= 2.22
|
# With git >= 2.22
|
||||||
# current_branch = git("branch --show-current", in_folder=app_cache_folder(app))
|
# current_branch = git("branch --show-current", in_folder=app_cache_folder(app))
|
||||||
current_branch = git("rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app))
|
current_branch = git(
|
||||||
|
"rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app)
|
||||||
|
)
|
||||||
if current_branch != branch:
|
if current_branch != branch:
|
||||||
# With git >= 2.13
|
# With git >= 2.13
|
||||||
# all_branches = git("branch --format=%(refname:short)", in_folder=app_cache_folder(app)).split()
|
# all_branches = git("branch --format=%(refname:short)", in_folder=app_cache_folder(app)).split()
|
||||||
all_branches = git("branch", in_folder=app_cache_folder(app)).split()
|
all_branches = git("branch", in_folder=app_cache_folder(app)).split()
|
||||||
all_branches.remove('*')
|
all_branches.remove("*")
|
||||||
if branch not in all_branches:
|
if branch not in all_branches:
|
||||||
git("remote set-branches --add origin %s" % branch, in_folder=app_cache_folder(app))
|
git(
|
||||||
git("fetch origin %s:%s" % (branch, branch), in_folder=app_cache_folder(app))
|
"remote set-branches --add origin %s" % branch,
|
||||||
|
in_folder=app_cache_folder(app),
|
||||||
|
)
|
||||||
|
git(
|
||||||
|
"fetch origin %s:%s" % (branch, branch),
|
||||||
|
in_folder=app_cache_folder(app),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
git("checkout --force %s" % branch, in_folder=app_cache_folder(app))
|
git("checkout --force %s" % branch, in_folder=app_cache_folder(app))
|
||||||
|
|
||||||
|
|
@ -119,7 +139,10 @@ def refresh_cache(app, infos):
|
||||||
except:
|
except:
|
||||||
# Sometimes there are tmp issue such that the refresh cache ..
|
# Sometimes there are tmp issue such that the refresh cache ..
|
||||||
# we don't trigger an error unless the cache hasnt been updated since more than 24 hours
|
# we don't trigger an error unless the cache hasnt been updated since more than 24 hours
|
||||||
if os.path.exists(fetch_head) and now - os.path.getmtime(fetch_head) < 24*3600:
|
if (
|
||||||
|
os.path.exists(fetch_head)
|
||||||
|
and now - os.path.getmtime(fetch_head) < 24 * 3600
|
||||||
|
):
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
raise
|
raise
|
||||||
|
|
@ -129,6 +152,7 @@ def refresh_cache(app, infos):
|
||||||
# Actual list build management #
|
# Actual list build management #
|
||||||
################################
|
################################
|
||||||
|
|
||||||
|
|
||||||
def build_catalog():
|
def build_catalog():
|
||||||
|
|
||||||
result_dict = {}
|
result_dict = {}
|
||||||
|
|
@ -151,33 +175,71 @@ def build_catalog():
|
||||||
categories = yaml.load(open("categories.yml").read())
|
categories = yaml.load(open("categories.yml").read())
|
||||||
antifeatures = yaml.load(open("antifeatures.yml").read())
|
antifeatures = yaml.load(open("antifeatures.yml").read())
|
||||||
os.system("mkdir -p ./builds/default/v2/")
|
os.system("mkdir -p ./builds/default/v2/")
|
||||||
with open("builds/default/v2/apps.json", 'w') as f:
|
with open("builds/default/v2/apps.json", "w") as f:
|
||||||
f.write(json.dumps({"apps": result_dict, "categories": categories, "antifeatures": antifeatures}, sort_keys=True))
|
f.write(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"apps": result_dict,
|
||||||
|
"categories": categories,
|
||||||
|
"antifeatures": antifeatures,
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
####################
|
####################
|
||||||
# Legacy version 1 #
|
# Legacy version 1 #
|
||||||
####################
|
####################
|
||||||
os.system("mkdir -p ./builds/default/v1/")
|
os.system("mkdir -p ./builds/default/v1/")
|
||||||
with open("./builds/default/v1/apps.json", 'w') as f:
|
with open("./builds/default/v1/apps.json", "w") as f:
|
||||||
f.write(json.dumps(result_dict, sort_keys=True))
|
f.write(json.dumps(result_dict, sort_keys=True))
|
||||||
|
|
||||||
####################
|
####################
|
||||||
# Legacy version 0 #
|
# Legacy version 0 #
|
||||||
####################
|
####################
|
||||||
official_apps = set(["agendav", "ampache", "baikal", "dokuwiki", "etherpad_mypads", "hextris", "jirafeau", "kanboard", "my_webapp", "nextcloud", "opensondage", "phpmyadmin", "piwigo", "rainloop", "roundcube", "searx", "shellinabox", "strut", "synapse", "transmission", "ttrss", "wallabag2", "wordpress", "zerobin"])
|
official_apps = set(
|
||||||
|
[
|
||||||
|
"agendav",
|
||||||
|
"ampache",
|
||||||
|
"baikal",
|
||||||
|
"dokuwiki",
|
||||||
|
"etherpad_mypads",
|
||||||
|
"hextris",
|
||||||
|
"jirafeau",
|
||||||
|
"kanboard",
|
||||||
|
"my_webapp",
|
||||||
|
"nextcloud",
|
||||||
|
"opensondage",
|
||||||
|
"phpmyadmin",
|
||||||
|
"piwigo",
|
||||||
|
"rainloop",
|
||||||
|
"roundcube",
|
||||||
|
"searx",
|
||||||
|
"shellinabox",
|
||||||
|
"strut",
|
||||||
|
"synapse",
|
||||||
|
"transmission",
|
||||||
|
"ttrss",
|
||||||
|
"wallabag2",
|
||||||
|
"wordpress",
|
||||||
|
"zerobin",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps}
|
official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps}
|
||||||
community_apps_dict = {k: v for k, v in result_dict.items() if k not in official_apps}
|
community_apps_dict = {
|
||||||
|
k: v for k, v in result_dict.items() if k not in official_apps
|
||||||
|
}
|
||||||
|
|
||||||
# We need the official apps to have "validated" as state to be recognized as official
|
# We need the official apps to have "validated" as state to be recognized as official
|
||||||
for app, infos in official_apps_dict.items():
|
for app, infos in official_apps_dict.items():
|
||||||
infos["state"] = "validated"
|
infos["state"] = "validated"
|
||||||
|
|
||||||
os.system("mkdir -p ./builds/default/v0/")
|
os.system("mkdir -p ./builds/default/v0/")
|
||||||
with open("./builds/default/v0/official.json", 'w') as f:
|
with open("./builds/default/v0/official.json", "w") as f:
|
||||||
f.write(json.dumps(official_apps_dict, sort_keys=True))
|
f.write(json.dumps(official_apps_dict, sort_keys=True))
|
||||||
|
|
||||||
with open("./builds/default/v0/community.json", 'w') as f:
|
with open("./builds/default/v0/community.json", "w") as f:
|
||||||
f.write(json.dumps(community_apps_dict, sort_keys=True))
|
f.write(json.dumps(community_apps_dict, sort_keys=True))
|
||||||
|
|
||||||
##############################
|
##############################
|
||||||
|
|
@ -185,6 +247,7 @@ def build_catalog():
|
||||||
##############################
|
##############################
|
||||||
categories = yaml.load(open("categories.yml").read())
|
categories = yaml.load(open("categories.yml").read())
|
||||||
os.system("mkdir -p ./builds/default/doc_catalog")
|
os.system("mkdir -p ./builds/default/doc_catalog")
|
||||||
|
|
||||||
def infos_for_doc_catalog(infos):
|
def infos_for_doc_catalog(infos):
|
||||||
level = infos.get("level")
|
level = infos.get("level")
|
||||||
if not isinstance(level, int):
|
if not isinstance(level, int):
|
||||||
|
|
@ -202,10 +265,18 @@ def build_catalog():
|
||||||
"bad_quality": level <= 5,
|
"bad_quality": level <= 5,
|
||||||
"antifeatures": infos["antifeatures"],
|
"antifeatures": infos["antifeatures"],
|
||||||
}
|
}
|
||||||
result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]}
|
|
||||||
with open("builds/default/doc_catalog/apps.json", 'w') as f:
|
|
||||||
f.write(json.dumps({"apps": result_dict_doc, "categories": categories}, sort_keys=True))
|
|
||||||
|
|
||||||
|
result_dict_doc = {
|
||||||
|
k: infos_for_doc_catalog(v)
|
||||||
|
for k, v in result_dict.items()
|
||||||
|
if v["state"] in ["working", "validated"]
|
||||||
|
}
|
||||||
|
with open("builds/default/doc_catalog/apps.json", "w") as f:
|
||||||
|
f.write(
|
||||||
|
json.dumps(
|
||||||
|
{"apps": result_dict_doc, "categories": categories}, sort_keys=True
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_app_dict(app, infos):
|
def build_app_dict(app, infos):
|
||||||
|
|
@ -214,42 +285,62 @@ def build_app_dict(app, infos):
|
||||||
this_app_cache = app_cache_folder(app)
|
this_app_cache = app_cache_folder(app)
|
||||||
assert os.path.exists(this_app_cache), "No cache yet for %s" % app
|
assert os.path.exists(this_app_cache), "No cache yet for %s" % app
|
||||||
|
|
||||||
infos['branch'] = infos.get('branch', 'master')
|
infos["branch"] = infos.get("branch", "master")
|
||||||
infos['revision'] = infos.get('revision', 'HEAD')
|
infos["revision"] = infos.get("revision", "HEAD")
|
||||||
|
|
||||||
# If using head, find the most recent meaningful commit in logs
|
# If using head, find the most recent meaningful commit in logs
|
||||||
if infos["revision"] == "HEAD":
|
if infos["revision"] == "HEAD":
|
||||||
relevant_files = ["manifest.json", "config_panel.toml", "hooks/", "scripts/", "conf/", "sources/"]
|
relevant_files = [
|
||||||
most_recent_relevant_commit = "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files)
|
"manifest.json",
|
||||||
|
"config_panel.toml",
|
||||||
|
"hooks/",
|
||||||
|
"scripts/",
|
||||||
|
"conf/",
|
||||||
|
"sources/",
|
||||||
|
]
|
||||||
|
most_recent_relevant_commit = (
|
||||||
|
"rev-list --full-history --all -n 1 -- " + " ".join(relevant_files)
|
||||||
|
)
|
||||||
infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache)
|
infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache)
|
||||||
assert re.match(r"^[0-9a-f]+$", infos["revision"]), "Output was not a commit? '%s'" % infos["revision"]
|
assert re.match(r"^[0-9a-f]+$", infos["revision"]), (
|
||||||
|
"Output was not a commit? '%s'" % infos["revision"]
|
||||||
|
)
|
||||||
# Otherwise, validate commit exists
|
# Otherwise, validate commit exists
|
||||||
else:
|
else:
|
||||||
assert infos["revision"] in git("rev-list --all", in_folder=this_app_cache).split("\n"), "Revision ain't in history ? %s" % infos["revision"]
|
assert infos["revision"] in git(
|
||||||
|
"rev-list --all", in_folder=this_app_cache
|
||||||
|
).split("\n"), ("Revision ain't in history ? %s" % infos["revision"])
|
||||||
|
|
||||||
# Find timestamp corresponding to that commit
|
# Find timestamp corresponding to that commit
|
||||||
timestamp = git("show -s --format=%ct " + infos["revision"], in_folder=this_app_cache)
|
timestamp = git(
|
||||||
assert re.match(r"^[0-9]+$", timestamp), "Failed to get timestamp for revision ? '%s'" % timestamp
|
"show -s --format=%ct " + infos["revision"], in_folder=this_app_cache
|
||||||
|
)
|
||||||
|
assert re.match(r"^[0-9]+$", timestamp), (
|
||||||
|
"Failed to get timestamp for revision ? '%s'" % timestamp
|
||||||
|
)
|
||||||
timestamp = int(timestamp)
|
timestamp = int(timestamp)
|
||||||
|
|
||||||
# Build the dict with all the infos
|
# Build the dict with all the infos
|
||||||
manifest = json.load(open(this_app_cache + "/manifest.json"))
|
manifest = json.load(open(this_app_cache + "/manifest.json"))
|
||||||
return {'id':manifest["id"],
|
return {
|
||||||
'git': {
|
"id": manifest["id"],
|
||||||
'branch': infos['branch'],
|
"git": {
|
||||||
'revision': infos["revision"],
|
"branch": infos["branch"],
|
||||||
'url': infos["url"]
|
"revision": infos["revision"],
|
||||||
|
"url": infos["url"],
|
||||||
},
|
},
|
||||||
'lastUpdate': timestamp,
|
"lastUpdate": timestamp,
|
||||||
'manifest': include_translations_in_manifest(manifest),
|
"manifest": include_translations_in_manifest(manifest),
|
||||||
'state': infos['state'],
|
"state": infos["state"],
|
||||||
'level': infos.get('level', '?'),
|
"level": infos.get("level", "?"),
|
||||||
'maintained': infos.get("maintained", True),
|
"maintained": infos.get("maintained", True),
|
||||||
'high_quality': infos.get("high_quality", False),
|
"high_quality": infos.get("high_quality", False),
|
||||||
'featured': infos.get("featured", False),
|
"featured": infos.get("featured", False),
|
||||||
'category': infos.get('category', None),
|
"category": infos.get("category", None),
|
||||||
'subtags': infos.get('subtags', []),
|
"subtags": infos.get("subtags", []),
|
||||||
'antifeatures': list(set(manifest.get('antifeatures', []) + infos.get('antifeatures', [])))
|
"antifeatures": list(
|
||||||
|
set(manifest.get("antifeatures", []) + infos.get("antifeatures", []))
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -274,14 +365,30 @@ def include_translations_in_manifest(manifest):
|
||||||
|
|
||||||
for category, questions in manifest["arguments"].items():
|
for category, questions in manifest["arguments"].items():
|
||||||
for question in questions:
|
for question in questions:
|
||||||
key = "%s_manifest_arguments_%s_%s" % (app_name, category, question["name"])
|
key = "%s_manifest_arguments_%s_%s" % (
|
||||||
|
app_name,
|
||||||
|
category,
|
||||||
|
question["name"],
|
||||||
|
)
|
||||||
# don't overwrite already existing translation in manifests for now
|
# don't overwrite already existing translation in manifests for now
|
||||||
if translations.get(key) and "ask" in question and current_lang not in question["ask"]:
|
if (
|
||||||
|
translations.get(key)
|
||||||
|
and "ask" in question
|
||||||
|
and current_lang not in question["ask"]
|
||||||
|
):
|
||||||
question["ask"][current_lang] = translations[key]
|
question["ask"][current_lang] = translations[key]
|
||||||
|
|
||||||
key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"])
|
key = "%s_manifest_arguments_%s_help_%s" % (
|
||||||
|
app_name,
|
||||||
|
category,
|
||||||
|
question["name"],
|
||||||
|
)
|
||||||
# don't overwrite already existing translation in manifests for now
|
# don't overwrite already existing translation in manifests for now
|
||||||
if translations.get(key) and "help" in question and current_lang not in question.get("help", []):
|
if (
|
||||||
|
translations.get(key)
|
||||||
|
and "help" in question
|
||||||
|
and current_lang not in question.get("help", [])
|
||||||
|
):
|
||||||
question["help"][current_lang] = translations[key]
|
question["help"][current_lang] = translations[key]
|
||||||
|
|
||||||
return manifest
|
return manifest
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue