mirror of
https://github.com/YunoHost/apps.git
synced 2024-09-03 20:06:07 +02:00
289 lines
11 KiB
Python
Executable file
289 lines
11 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import json
|
|
import subprocess
|
|
import yaml
|
|
import time
|
|
|
|
now = time.time()
|
|
|
|
catalog = json.load(open("apps.json"))
|
|
catalog = {app: infos for app, infos in catalog.items() if infos.get('state') != 'notworking'}
|
|
|
|
my_env = os.environ.copy()
|
|
my_env["GIT_TERMINAL_PROMPT"] = "0"
|
|
|
|
os.makedirs(".apps_cache", exist_ok=True)
|
|
os.makedirs("builds/", exist_ok=True)
|
|
|
|
def error(msg):
|
|
msg = "[Applist builder error] " + msg
|
|
if os.path.exists("/usr/bin/sendxmpppy"):
|
|
subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, 'wb'))
|
|
print(msg + "\n")
|
|
|
|
# Progress bar helper, stolen from https://stackoverflow.com/a/34482761
|
|
def progressbar(it, prefix="", size=60, file=sys.stdout):
|
|
count = len(it)
|
|
def show(j, name=""):
|
|
name += " "
|
|
x = int(size*j/count)
|
|
file.write("%s[%s%s] %i/%i %s\r" % (prefix, "#"*x, "."*(size-x), j, count, name))
|
|
file.flush()
|
|
show(0)
|
|
for i, item in enumerate(it):
|
|
yield item
|
|
show(i+1, item[0])
|
|
file.write("\n")
|
|
file.flush()
|
|
|
|
###################################
|
|
# App git clones cache management #
|
|
###################################
|
|
|
|
def app_cache_folder(app):
|
|
return os.path.join(".apps_cache", app)
|
|
|
|
|
|
def git(cmd, in_folder=None):
|
|
|
|
if in_folder:
|
|
cmd = "-C " + in_folder + " " + cmd
|
|
cmd = "git " + cmd
|
|
return subprocess.check_output(cmd.split(), env=my_env).strip().decode("utf-8")
|
|
|
|
|
|
def refresh_all_caches():
|
|
|
|
for app, infos in progressbar(sorted(catalog.items()), "Updating git clones: ", 40):
|
|
app = app.lower()
|
|
if not os.path.exists(app_cache_folder(app)):
|
|
try:
|
|
init_cache(app, infos)
|
|
except Exception as e:
|
|
error("Failed to init cache for %s" % app)
|
|
else:
|
|
try:
|
|
refresh_cache(app, infos)
|
|
except Exception as e:
|
|
error("Failed to not refresh cache for %s" % app)
|
|
|
|
|
|
def init_cache(app, infos):
|
|
|
|
if infos["state"] == "notworking":
|
|
depth = 5
|
|
if infos["state"] == "inprogress":
|
|
depth = 20
|
|
else:
|
|
depth = 40
|
|
|
|
git("clone --quiet --depth {depth} --single-branch --branch {branch} {url} {folder}".format(
|
|
depth=depth,
|
|
url=infos["url"],
|
|
branch=infos.get("branch", "master"),
|
|
folder=app_cache_folder(app))
|
|
)
|
|
|
|
|
|
def refresh_cache(app, infos):
|
|
|
|
# Don't refresh if already refreshed during last hour
|
|
fetch_head = app_cache_folder(app) + "/.git/FETCH_HEAD"
|
|
if os.path.exists(fetch_head) and now - os.path.getmtime(fetch_head) < 3600:
|
|
return
|
|
|
|
branch = infos.get("branch", "master")
|
|
|
|
try:
|
|
git("remote set-url origin " + infos["url"], in_folder=app_cache_folder(app))
|
|
# With git >= 2.22
|
|
# current_branch = git("branch --show-current", in_folder=app_cache_folder(app))
|
|
current_branch = git("rev-parse --abbrev-ref HEAD", in_folder=app_cache_folder(app))
|
|
if current_branch != branch:
|
|
# With git >= 2.13
|
|
# all_branches = git("branch --format=%(refname:short)", in_folder=app_cache_folder(app)).split()
|
|
all_branches = git("branch", in_folder=app_cache_folder(app)).split()
|
|
all_branches.remove('*')
|
|
if branch not in all_branches:
|
|
git("remote set-branches --add origin %s" % branch, in_folder=app_cache_folder(app))
|
|
git("fetch origin %s:%s" % (branch, branch), in_folder=app_cache_folder(app))
|
|
else:
|
|
git("checkout --force %s" % branch, in_folder=app_cache_folder(app))
|
|
|
|
git("fetch --quiet origin %s --force" % branch, in_folder=app_cache_folder(app))
|
|
git("reset origin/%s --hard" % branch, in_folder=app_cache_folder(app))
|
|
except:
|
|
# Sometimes there are tmp issue such that the refresh cache ..
|
|
# we don't trigger an error unless the cache hasnt been updated since more than 24 hours
|
|
if os.path.exists(fetch_head) and now - os.path.getmtime(fetch_head) < 24*3600:
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
|
|
################################
|
|
# Actual list build management #
|
|
################################
|
|
|
|
def build_catalog():
|
|
|
|
result_dict = {}
|
|
|
|
for app, infos in progressbar(sorted(catalog.items()), "Processing: ", 40):
|
|
|
|
app = app.lower()
|
|
|
|
try:
|
|
app_dict = build_app_dict(app, infos)
|
|
except Exception as e:
|
|
error("Processing %s failed: %s" % (app, str(e)))
|
|
continue
|
|
|
|
result_dict[app_dict["id"]] = app_dict
|
|
|
|
#####################
|
|
# Current version 2 #
|
|
#####################
|
|
categories = yaml.load(open("categories.yml").read())
|
|
os.system("mkdir -p ./builds/default/v2/")
|
|
with open("builds/default/v2/apps.json", 'w') as f:
|
|
f.write(json.dumps({"apps": result_dict, "categories": categories}, sort_keys=True))
|
|
|
|
####################
|
|
# Legacy version 1 #
|
|
####################
|
|
os.system("mkdir -p ./builds/default/v1/")
|
|
with open("./builds/default/v1/apps.json", 'w') as f:
|
|
f.write(json.dumps(result_dict, sort_keys=True))
|
|
|
|
####################
|
|
# Legacy version 0 #
|
|
####################
|
|
official_apps = set(["agendav", "ampache", "baikal", "dokuwiki", "etherpad_mypads", "hextris", "jirafeau", "kanboard", "my_webapp", "nextcloud", "opensondage", "phpmyadmin", "piwigo", "rainloop", "roundcube", "searx", "shellinabox", "strut", "synapse", "transmission", "ttrss", "wallabag2", "wordpress", "zerobin"])
|
|
|
|
official_apps_dict = {k: v for k, v in result_dict.items() if k in official_apps}
|
|
community_apps_dict = {k: v for k, v in result_dict.items() if k not in official_apps}
|
|
|
|
# We need the official apps to have "validated" as state to be recognized as official
|
|
for app, infos in official_apps_dict.items():
|
|
infos["state"] = "validated"
|
|
|
|
os.system("mkdir -p ./builds/default/v0/")
|
|
with open("./builds/default/v0/official.json", 'w') as f:
|
|
f.write(json.dumps(official_apps_dict, sort_keys=True))
|
|
|
|
with open("./builds/default/v0/community.json", 'w') as f:
|
|
f.write(json.dumps(community_apps_dict, sort_keys=True))
|
|
|
|
##############################
|
|
# Version for catalog in doc #
|
|
##############################
|
|
categories = yaml.load(open("categories.yml").read())
|
|
os.system("mkdir -p ./builds/default/doc_catalog")
|
|
def infos_for_doc_catalog(infos):
|
|
level = infos.get("level")
|
|
if not isinstance(level, int):
|
|
level = -1
|
|
return {
|
|
"id": infos["id"],
|
|
"category": infos["category"],
|
|
"url": infos["git"]["url"],
|
|
"name": infos["manifest"]["name"],
|
|
"description": infos["manifest"]["description"],
|
|
"state": infos["state"],
|
|
"level": level,
|
|
"broken": level <= 0,
|
|
"good_quality": level >= 8,
|
|
"bad_quality": level <= 5,
|
|
}
|
|
result_dict_doc = {k: infos_for_doc_catalog(v) for k, v in result_dict.items() if v["state"] in ["working", "validated"]}
|
|
with open("builds/default/doc_catalog/apps.json", 'w') as f:
|
|
f.write(json.dumps({"apps": result_dict_doc, "categories": categories}, sort_keys=True))
|
|
|
|
|
|
|
|
def build_app_dict(app, infos):
|
|
|
|
# Make sure we have some cache
|
|
this_app_cache = app_cache_folder(app)
|
|
assert os.path.exists(this_app_cache), "No cache yet for %s" % app
|
|
|
|
# If using head, find the most recent meaningful commit in logs
|
|
if infos["revision"] == "HEAD":
|
|
relevant_files = ["manifest.json", "actions.json", "hooks/", "scripts/", "conf/", "sources/"]
|
|
most_recent_relevant_commit = "rev-list --full-history --all -n 1 -- " + " ".join(relevant_files)
|
|
infos["revision"] = git(most_recent_relevant_commit, in_folder=this_app_cache)
|
|
assert re.match(r"^[0-9a-f]+$", infos["revision"]), "Output was not a commit? '%s'" % infos["revision"]
|
|
# Otherwise, validate commit exists
|
|
else:
|
|
assert infos["revision"] in git("rev-list --all", in_folder=this_app_cache).split("\n"), "Revision ain't in history ? %s" % infos["revision"]
|
|
|
|
# Find timestamp corresponding to that commit
|
|
timestamp = git("show -s --format=%ct " + infos["revision"], in_folder=this_app_cache)
|
|
assert re.match(r"^[0-9]+$", timestamp), "Failed to get timestamp for revision ? '%s'" % timestamp
|
|
timestamp = int(timestamp)
|
|
|
|
# Build the dict with all the infos
|
|
manifest = json.load(open(this_app_cache + "/manifest.json"))
|
|
return {'id':manifest["id"],
|
|
'git': {
|
|
'branch': infos['branch'],
|
|
'revision': infos["revision"],
|
|
'url': infos["url"]
|
|
},
|
|
'lastUpdate': timestamp,
|
|
'manifest': include_translations_in_manifest(manifest),
|
|
'state': infos['state'],
|
|
'level': infos.get('level', '?'),
|
|
'maintained': infos.get("maintained", True),
|
|
'high_quality': infos.get("high_quality", False),
|
|
'featured': infos.get("featured", False),
|
|
'category': infos.get('category', None),
|
|
'subtags': infos.get('subtags', []),
|
|
}
|
|
|
|
|
|
def include_translations_in_manifest(manifest):
|
|
|
|
app_name = manifest["id"]
|
|
|
|
for locale in os.listdir("locales"):
|
|
if not locale.endswith("json"):
|
|
continue
|
|
|
|
if locale == "en.json":
|
|
continue
|
|
|
|
current_lang = locale.split(".")[0]
|
|
translations = json.load(open(os.path.join("locales", locale), "r"))
|
|
|
|
key = "%s_manifest_description" % app_name
|
|
if translations.get(key, None):
|
|
manifest["description"][current_lang] = translations[key]
|
|
|
|
for category, questions in manifest["arguments"].items():
|
|
for question in questions:
|
|
key = "%s_manifest_arguments_%s_%s" % (app_name, category, question["name"])
|
|
# don't overwrite already existing translation in manifests for now
|
|
if translations.get(key) and "ask" in question and not current_lang not in question["ask"]:
|
|
#print("[ask]", current_lang, key)
|
|
question["ask"][current_lang] = translations[key]
|
|
|
|
key = "%s_manifest_arguments_%s_help_%s" % (app_name, category, question["name"])
|
|
# don't overwrite already existing translation in manifests for now
|
|
if translations.get(key) and not current_lang not in question.get("help", []):
|
|
#print("[help]", current_lang, key)
|
|
question["help"][current_lang] = translations[key]
|
|
|
|
return manifest
|
|
|
|
|
|
######################
|
|
|
|
if __name__ == "__main__":
|
|
refresh_all_caches()
|
|
build_catalog()
|