diff --git a/manifestv2_app_ynh/convert_to_packaging_format_v2.py b/manifestv2_app_ynh/convert_to_packaging_format_v2.py deleted file mode 100644 index 4e7b2f1..0000000 --- a/manifestv2_app_ynh/convert_to_packaging_format_v2.py +++ /dev/null @@ -1,146 +0,0 @@ -import copy -import json - - -def _convert_v1_manifest_to_v2(manifest): - - manifest = copy.deepcopy(manifest) - - if "upstream" not in manifest: - manifest["upstream"] = {} - - if "license" in manifest and "license" not in manifest["upstream"]: - manifest["upstream"]["license"] = manifest["license"] - - if "url" in manifest and "website" not in manifest["upstream"]: - manifest["upstream"]["website"] = manifest["url"] - - manifest["integration"] = { - "yunohost": manifest.get("requirements", {}).get("yunohost"), - "architectures": "all", - "multi_instance": manifest.get("multi_instance", False), - "ldap": "?", - "sso": "?", - } - - maintainer = manifest.get("maintainer", {}).get("name") - manifest["maintainers"] = [maintainer] if maintainer else [] - - install_questions = manifest["arguments"]["install"] - manifest["install"] = {} - for question in install_questions: - name = question.pop("name") - if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: - question.pop("ask") - if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: - question.pop("example") - - manifest["install"][name] = question - - manifest["resources"] = { - "disk": { - "build": "50M", # This is an *estimate* minimum value for the disk needed at build time (e.g. during install/upgrade) and during regular usage - "usage": "50M" # Please only use round values such as: 10M, 100M, 200M, 400M, 800M, 1G, 2G, 4G, 8G - }, - "ram": { - "build": "50M", # This is an *estimate* minimum value for the RAM needed at build time (i.e. during install/upgrade) and during regular usage - "usage": "10M", # Please only use round values like ["10M", "100M", "200M", "400M", "800M", "1G", "2G", "4G", "8G"] - "include_swap": False - }, - "route": {}, - "install_dir": { - "base_dir": "/var/www/", # This means that the app shall be installed in /var/www/$app which is the standard for webapps. You may change this to /opt/ if the app is a system app. - "alias": "final_path" - } - } - - if "domain" in manifest["install"] and "path" in manifest["install"]: - manifest["resources"]["route"]["url"] = "{domain}{path}" - elif "path" not in manifest["install"]: - manifest["resources"]["route"]["url"] = "{domain}/" - else: - del manifest["resources"]["route"] - - keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] - - keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] - for key in keys_to_del: - del manifest[key] - - return manifest - - -def _dump_v2_manifest_as_toml(manifest): - - import re - from tomlkit import document, nl, table, dumps, comment - - toml_manifest = document() - toml_manifest.add("packaging_format", 2) - toml_manifest.add(nl()) - toml_manifest.add("id", manifest["id"]) - toml_manifest.add("name", manifest["name"]) - for lang, value in manifest["description"].items(): - toml_manifest.add(f"description.{lang}", value) - toml_manifest.add(nl()) - toml_manifest.add("version", manifest["version"]) - toml_manifest.add(nl()) - toml_manifest.add("maintainers", manifest["maintainers"]) - - upstream = table() - for key, value in manifest["upstream"].items(): - upstream[key] = value - toml_manifest["upstream"] = upstream - - integration = table() - for key, value in manifest["integration"].items(): - integration[key] = value - integration["architectures"].comment('Can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature, for example: ["amd64", "i386"]') - toml_manifest["integration"] = integration - - install = table() - for key, value in manifest["install"].items(): - install[key] = table() - install[key].indent(4) - - if key in ["domain", "path", "admin", "is_public", "password"]: - install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core")) - - for lang, value2 in value.get("ask", {}).items(): - install[key].add(f"ask.{lang}", value2) - - for lang, value2 in value.get("help", {}).items(): - install[key].add(f"help.{lang}", value2) - - for key2, value2 in value.items(): - if key2 in ["ask", "help"]: - continue - install[key].add(key2, value2) - - toml_manifest["install"] = install - - resources = table() - for key, value in manifest["resources"].items(): - resources[key] = table() - resources[key].indent(4) - for key2, value2 in value.items(): - resources[key].add(key2, value2) - if key in ["disk", "ram"] and key2 == "build": - resources[key][key2].comment(f"This is an *estimate* minimum value for the {key} needed at build time (e.g. during install/upgrade) and during regular usage") - elif key in ["disk", "ram"] and key2 == "usage": - resources[key][key2].comment("Please only use round values such as: 10M, 100M, 200M, 400M, 800M, 1G, 2G, 4G, 8G") - elif key == "install_dir" and key2 == "base_dir": - resources[key][key2].comment("This means that the app shall be installed in /var/www/$app which is the standard for webapps. You may change this to /opt/ if the app is a system app.") - - toml_manifest["resources"] = resources - - toml_manifest_dump = dumps(toml_manifest) - - regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"') - toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump) - return toml_manifest_dump - - -manifest = json.load(open("manifest.json")) -manifest = _convert_v1_manifest_to_v2(manifest) -print(_dump_v2_manifest_as_toml(manifest)) diff --git a/manifestv2_app_ynh/convert_to_packaging_v2.py b/manifestv2_app_ynh/convert_to_packaging_v2.py new file mode 100644 index 0000000..5603955 --- /dev/null +++ b/manifestv2_app_ynh/convert_to_packaging_v2.py @@ -0,0 +1,310 @@ +import os +import re +import json +import subprocess + + +def check_output(cmd): + return ( + subprocess.check_output(cmd, shell=True) + .decode("utf-8") + .strip() + ) + + +def _convert_v1_manifest_to_v2(app_path): + + manifest = json.load(open(app_path + "/manifest.json")) + + if "upstream" not in manifest: + manifest["upstream"] = {} + + if "license" in manifest and "license" not in manifest["upstream"]: + manifest["upstream"]["license"] = manifest["license"] + + if "url" in manifest and "website" not in manifest["upstream"]: + manifest["upstream"]["website"] = manifest["url"] + + manifest["integration"] = { + "yunohost": manifest.get("requirements", {}).get("yunohost"), + "architectures": "all", + "multi_instance": manifest.get("multi_instance", False), + "ldap": "?", + "sso": "?", + "disk": "50M", + "ram.build": "50M", + "ram.runtime": "50M" + } + + maintainer = manifest.get("maintainer", {}).get("name") + manifest["maintainers"] = [maintainer] if maintainer else [] + + install_questions = manifest["arguments"]["install"] + manifest["install"] = {} + for question in install_questions: + name = question.pop("name") + if "ask" in question and name in ["domain", "path", "admin", "is_public", "password"]: + question.pop("ask") + if question.get("example") and question.get("type") in ["domain", "path", "user", "boolean", "password"]: + question.pop("example") + + manifest["install"][name] = question + + # Rename is_public to init_main_permission + manifest["install"] = {(k if k != "is_public" else "init_main_permission"): v for k, v in manifest["install"].items()} + + if "init_main_permission" in manifest["install"]: + manifest["install"]["init_main_permission"]["type"] = "group" + if manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "visitors" + elif manifest["install"]["init_main_permission"].get("default") is True: + manifest["install"]["init_main_permission"]["default"] = "all_users" + + if "domain" in manifest["install"] and "path" not in manifest["install"]: + manifest["install"]["domain"]["full_domain"] = True + + manifest["resources"] = {} + manifest["resources"]["system_user"] = {} + manifest["resources"]["install_dir"] = {} + + if os.system(f"grep -q 'datadir=' {app_path}/scripts/install") == 0: + manifest["resources"]["data_dir"] = {} + + manifest["resources"]["permissions"] = {} + + if os.system(f"grep -q 'ynh_webpath_register' '{app_path}/scripts/install'") == 0: + manifest["resources"]["permissions"]["main.url"] = "/" + + # FIXME: Parse ynh_permission_create --permission="admin" --url="/wp-login.php" --additional_urls="/wp-admin.php" --allowed=$admin_wordpress + + ports = check_output(f"sed -nr 's/(\\w+)=.*ynh_find_port[^0-9]*([0-9]+)\\)/\\1,\\2/p' '{app_path}/scripts/install'") + if ports: + manifest["resources"]["ports"] = {} + for port in ports.split("\n"): + name, default = port.split(",") + exposed = check_output(f"sed -nr 's/.*yunohost firewall allow .*(TCP|UDP|Both).*${name}/\\1/p' '{app_path}/scripts/install'") + if exposed == "Both": + exposed = True + + name = name.replace("_port", "").replace("port_", "") + if name == "port": + name = "main" + + manifest["resources"]["ports"][f"{name}.default"] = int(default) + if exposed: + manifest["resources"]["ports"][f"{name}.exposed"] = exposed + + maybequote = "[\"'\"'\"']?" + apt_dependencies = check_output(f"sed -nr 's/.*_dependencies={maybequote}(.*){maybequote}? *$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d '\"' | sed 's@ @\\n@g'") + php_version = check_output(f"sed -nr 's/^ *YNH_PHP_VERSION={maybequote}(.*){maybequote}?$/\\1/p' '{app_path}/scripts/_common.sh' | tr -d \"\\\"'\"") + if apt_dependencies.strip(): + if php_version: + apt_dependencies = apt_dependencies.replace("${YNH_PHP_VERSION}", php_version) + apt_dependencies = ', '.join([d for d in apt_dependencies.split("\n") if d]) + manifest["resources"]["apt"] = {"packages": apt_dependencies} + + extra_apt_repos = check_output(r"sed -nr 's/.*_extra_app_dependencies.*repo=\"(.*)\".*package=\"(.*)\".*key=\"(.*)\"/\1,\2,\3/p' %s/scripts/install" % app_path) + if extra_apt_repos: + for i, extra_apt_repo in enumerate(extra_apt_repos.split("\n")): + repo, packages, key = extra_apt_repo.split(",") + packages = packages.replace('$', '#FIXME#$') + if "apt" not in manifest["resources"]: + manifest["resources"]["apt"] = {} + if "extras" not in manifest["resources"]["apt"]: + manifest["resources"]["apt"]["extras"] = [] + manifest["resources"]["apt"]["extras"].append({ + "repo": repo, + "key": key, + "packages": packages, + }) + + if os.system(f"grep -q 'ynh_mysql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "mysql"} + elif os.system(f"grep -q 'ynh_psql_setup_db' {app_path}/scripts/install") == 0: + manifest["resources"]["database"] = {"type": "postgresql"} + + keys_to_keep = ["packaging_format", "id", "name", "description", "version", "maintainers", "upstream", "integration", "install", "resources"] + + keys_to_del = [key for key in manifest.keys() if key not in keys_to_keep] + for key in keys_to_del: + del manifest[key] + + return manifest + + +def _dump_v2_manifest_as_toml(manifest): + + import re + from tomlkit import document, nl, table, dumps, comment + + toml_manifest = document() + toml_manifest.add("packaging_format", 2) + toml_manifest.add(nl()) + toml_manifest.add("id", manifest["id"]) + toml_manifest.add("name", manifest["name"]) + for lang, value in manifest["description"].items(): + toml_manifest.add(f"description.{lang}", value) + toml_manifest.add(nl()) + toml_manifest.add("version", manifest["version"]) + toml_manifest.add(nl()) + toml_manifest.add("maintainers", manifest["maintainers"]) + + upstream = table() + for key, value in manifest["upstream"].items(): + upstream[key] = value + toml_manifest["upstream"] = upstream + + integration = table() + for key, value in manifest["integration"].items(): + integration.add(key, value) + integration["architectures"].comment('FIXME: can be replaced by a list of supported archs using the dpkg --print-architecture nomenclature (amd64/i386/armhf/arm64/armel), for example: ["amd64", "i386"]') + integration["ldap"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["sso"].comment('FIXME: replace with true, false, or "not_relevant"') + integration["disk"].comment('FIXME: replace with an **estimate** minimum disk requirement. e.g. 20M, 400M, 1G, ...') + integration["ram.build"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + integration["ram.runtime"].comment('FIXME: replace with an **estimate** minimum ram requirement. e.g. 50M, 400M, 1G, ...') + toml_manifest["integration"] = integration + + install = table() + for key, value in manifest["install"].items(): + install[key] = table() + install[key].indent(4) + + if key in ["domain", "path", "admin", "is_public", "password"]: + install[key].add(comment("this is a generic question - ask strings are automatically handled by Yunohost's core")) + + for lang, value2 in value.get("ask", {}).items(): + install[key].add(f"ask.{lang}", value2) + + for lang, value2 in value.get("help", {}).items(): + install[key].add(f"help.{lang}", value2) + + for key2, value2 in value.items(): + if key2 in ["ask", "help"]: + continue + install[key].add(key2, value2) + + toml_manifest["install"] = install + + resources = table() + for key, value in manifest["resources"].items(): + resources[key] = table() + resources[key].indent(4) + for key2, value2 in value.items(): + resources[key].add(key2, value2) + if key == "apt" and key2 == "extras": + for extra in resources[key][key2]: + extra.indent(8) + + toml_manifest["resources"] = resources + + toml_manifest_dump = dumps(toml_manifest) + + regex = re.compile(r'\"((description|ask|help)\.[a-z]{2})\"') + toml_manifest_dump = regex.sub(r'\1', toml_manifest_dump) + toml_manifest_dump = toml_manifest_dump.replace('"ram.build"', "ram.build") + toml_manifest_dump = toml_manifest_dump.replace('"ram.runtime"', "ram.runtime") + toml_manifest_dump = toml_manifest_dump.replace('"main.url"', "main.url") + toml_manifest_dump = toml_manifest_dump.replace('"main.default"', "main.default") + return toml_manifest_dump + + +def cleanup_scripts_and_conf(folder): + + patterns_to_remove_in_scripts = [ + "^.*ynh_abort_if_errors.*$", + "^.*YNH_APP_ARG.*$", + "^.*YNH_APP_INSTANCE_NAME.*$", + r"^ *final_path=", + r"^\s*final_path=", + "^.*test .*-(e|d) .*final_path.*$", + "^.*ynh_webpath_register.*$", + "^.*ynh_webpath_available.*$", + "^.*ynh_system_user_create.*$", + "^.*ynh_system_user_delete.*$", + "^.*ynh_permission_update.*$", + "^.*ynh_permission_create.*$", + "^.*if .*ynh_permission_exists.*$", + "^.*if .*ynh_legacy_permissions_exists.*$", + "^.*ynh_legacy_permissions_delete_all.*$", + "^.*ynh_app_setting_set .*(domain|path|final_path|admin|password|port|datadir|db_name|db_user|db_pwd).*$", + "^.*ynh_app_setting_.* is_public.*$", + r"^.*if.*\$is_public.*$", + "^.*_dependencies=.*$", + "^.*ynh_install_app_dependencies.*$", + "^.*ynh_install_extra_app_dependencies.*$", + "^.*ynh_remove_app_dependencies.*$", + r"^.*\$\(ynh_app_setting_get.*$", + r"^.*ynh_secure_remove .*\$final_path.*$", + r"^.*ynh_secure_remove .*\$datadir.*$", + "^.*ynh_backup_before_upgrade.*$", + "^.*ynh_clean_setup.*$", + "^.*ynh_restore_upgradebackup.*$", + "^db_name=.*$", + "^db_user=.*$", + "^db_pwd=.*$", + "^datadir=.*$", + "^.*ynh_psql_test_if_first_run.*$", + "^.*ynh_mysql_setup_db.*$", + "^.*ynh_psql_setup_db.*$", + "^.*ynh_mysql_remove_db.*$", + "^.*ynh_psql_remove_db.*$", + "^.*ynh_find_port.*$", + "^.*ynh_script_progression.*Finding an available port", + "^.*ynh_script_progression.*Backing up the app before upgrading", + "^.*ynh_script_progression.*Creating data directory", + "^.*ynh_script_progression.*system user", + "^.*ynh_script_progression.*installation settings", + "^.*ynh_print_info.*installation settings", + r"^.*ynh_script_progression.*\w+ dependencies", + "^.*ynh_script_progression.*Removing app main dir", + "^.*ynh_script_progression.*Validating.*parameters", + "^.*ynh_script_progression.*SQL database", + "^.*ynh_script_progression.*Configuring permissions", + ] + patterns_to_remove_in_scripts = [re.compile(f"({p})", re.MULTILINE) for p in patterns_to_remove_in_scripts] + + replaces = [ + ("path_url", "path"), + ("PATH_URL", "PATH"), + ("final_path", "install_dir"), + ("FINALPATH", "INSTALL_DIR"), + ("datadir", "data_dir"), + ("DATADIR", "DATA_DIR"), + ] + + for s in ["_common.sh", "install", "remove", "upgrade", "backup", "restore"]: + + script = f"{folder}/scripts/{s}" + + content = open(script).read() + + for pattern in patterns_to_remove_in_scripts: + content = pattern.sub(r"#REMOVEME? \1", content) + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + open(script, "w").write(content) + + for conf in os.listdir(f"{folder}/conf"): + + conf = f"{folder}/conf/{conf}" + + if not os.path.isfile(conf): + continue + + content = open(conf).read() + content_init = content + + for pattern, replace in replaces: + content = content.replace(pattern, replace) + + if content_init != content: + open(conf, "w").write(content) + + +manifest = _convert_v1_manifest_to_v2("./") +open("manifest.toml", "w").write(_dump_v2_manifest_as_toml(manifest)) + +cleanup_scripts_and_conf("./")