#!/usr/bin/env python3 # -*- coding: utf8 -*- import sys import os import re import json import shlex import urllib.request import codecs reader = codecs.getreader("utf-8") # ############################################################################ # Utilities # ############################################################################ # Taken from https://stackoverflow.com/a/49518779 def check_for_duplicate_keys(ordered_pairs): dict_out = {} for key, val in ordered_pairs: if key in dict_out: print_warning("Duplicated key '%s' in %s" % (key, ordered_pairs)) else: dict_out[key] = val return dict_out class c: HEADER = '\033[94m' OKBLUE = '\033[94m' OKGREEN = '\033[92m' WARNING = '\033[93m' MAYBE_FAIL = '\033[96m' FAIL = '\033[91m' END = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' def header(app): print(""" [{header}{bold}YunoHost App Package Linter{end}] App packaging documentation - https://yunohost.org/#/packaging_apps App package example - https://github.com/YunoHost/example_ynh Official helpers - https://yunohost.org/#/packaging_apps_helpers_en Experimental helpers - https://github.com/YunoHost-Apps/Experimental_helpers Analyzing package {header}{app}{end}""" .format(header=c.HEADER, bold=c.BOLD, end=c.END, app=app)) def print_header(str): print("\n [" + c.BOLD + c.HEADER + str.title() + c.END + "]\n") def print_warning_not_reliable(str): print(c.MAYBE_FAIL + "?", str, c.END) warning_count = 0 def print_warning(str): global warning_count warning_count += 1 print(c.WARNING + "!", str, c.END) error_count = 0 def print_error(str): global error_count error_count += 1 print(c.FAIL + "✘", str, c.END) def print_happy(str): print(c.OKGREEN + "☺ ", str, "♥") def urlopen(url): try: conn = urllib.request.urlopen(url) except urllib.error.HTTPError as e: return {'content': '', 'code': e.code} except urllib.error.URLError as e: print('URLError') return {'content': conn.read().decode('UTF8'), 'code': 200} def file_exists(file_path): return os.path.isfile(file_path) and os.stat(file_path).st_size > 0 def spdx_licenses(): cachefile = ".spdx_licenses" if os.path.exists(cachefile): return open(cachefile).read() link = "https://spdx.org/licenses/" content = urlopen(link)['content'] open(cachefile, "w").write(content) return content # ############################################################################ # Actual high-level checks # ############################################################################ scriptnames = ["_common.sh", "install", "remove", "upgrade", "backup", "restore"] class App(): def __init__(self, path): print_header("LOADING APP") self.path = path self.scripts = {f: Script(self.path, f) for f in scriptnames} def analyze(self): self.misc_file_checks() self.check_helper_consistency() self.check_source_management() self.check_manifest() for script in [self.scripts[s] for s in scriptnames if self.scripts[s].exists]: script.analyze() def misc_file_checks(self): print_header("MISC FILE CHECKS") # # Check for recommended and mandatory files # filenames = ("manifest.json", "LICENSE", "README.md", "scripts/install", "scripts/remove", "scripts/upgrade", "scripts/backup", "scripts/restore") non_mandatory = ("script/backup", "script/restore") for filename in filenames: if file_exists(self.path + "/" + filename): continue elif filename in non_mandatory: print_warning("Consider adding a file %s" % filename) else: print_error("Providing a %s is mandatory" % filename) # # Deprecated php-fpm.ini thing # if file_exists(self.path + "/conf/php-fpm.ini"): print_warning( "Using a separate php-fpm.ini file is deprecated. " "Please merge your php-fpm directives directly in the pool file. " "(c.f. https://github.com/YunoHost-Apps/nextcloud_ynh/issues/138 )" ) # # Analyze nginx conf # - Deprecated usage of 'add_header' in nginx conf # - Spot path traversal issue vulnerability # for filename in os.listdir(self.path + "/conf"): # Ignore subdirs or filename not containing nginx in the name if not os.path.isfile(self.path + "/conf/" + filename) or "nginx" not in filename: continue # # 'add_header' usage # content = open(self.path + "/conf/" + filename).read() if "location" in content and "add_header" in content: print_warning( "Do not use 'add_header' in the nginx conf. Use 'more_set_headers' instead. " "(See https://www.peterbe.com/plog/be-very-careful-with-your-add_header-in-nginx " "and https://github.com/openresty/headers-more-nginx-module#more_set_headers )" ) # # Path traversal issues # from lib.nginxparser import nginxparser nginxconf = nginxparser.load(open(self.path + "/conf/" + filename)) def find_location_with_alias(locationblock): if locationblock[0][0] != "location": return location = locationblock[0][-1] for line in locationblock[1]: instruction = line[0] if instruction == "alias": yield (location, line) elif isinstance(instruction, list) and instruction and instruction[0] == "location": yield from find_location_with_alias(instruction) else: continue def find_path_traversal_issue(nginxconf): for block in nginxconf: for location, alias in find_location_with_alias(block): alias_path = alias[-1] if not location.endswith("/") and alias_path.endswith("/"): yield location for location in find_path_traversal_issue(nginxconf): print_warning( "The nginx configuration (especially location %s) " "appears vulnerable to path traversal issues as explained in\n" " https://www.acunetix.com/vulnerabilities/web/path-traversal-via-misconfigured-nginx-alias/\n" "To fix it, look at the first lines of the nginx conf of the example app : \n" " https://github.com/YunoHost/example_ynh/blob/master/conf/nginx.conf" % location ) def check_helper_consistency(self): """ check if ynh_install_app_dependencies is present in install/upgrade/restore so dependencies are up to date after restoration or upgrade """ install_script = self.scripts["install"] if install_script.exists: if install_script.contains("ynh_install_app_dependencies"): for name in ["upgrade", "restore"]: if self.scripts[name].exists and not self.scripts[name].contains("ynh_install_app_dependencies"): print_warning("ynh_install_app_dependencies should also be in %s script" % name) if install_script.contains("yunohost service add"): if self.scripts["remove"].exists and not self.scripts["remove"].contains("yunohost service remove"): print_error( "You used 'yunohost service add' in the install script, " "but not 'yunohost service remove' in the remove script." ) def check_source_management(self): print_header("SOURCES MANAGEMENT") DIR = os.path.join(self.path, "sources") # Check if there is more than six files on 'sources' folder if os.path.exists(os.path.join(self.path, "sources")) \ and len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) > 5: print_warning( "[YEP-3.3] Upstream app sources shouldn't be stored in this 'sources' folder of this git repository as a copy/paste\n" "During installation, the package should download sources from upstream via 'ynh_setup_source'.\n" "See the helper documentation. " "Original discussion happened here : " "https://github.com/YunoHost/issues/issues/201#issuecomment-391549262" ) def check_manifest(self): manifest = os.path.join(self.path, 'manifest.json') if not os.path.exists(manifest): return print_header("MANIFEST") """ Check if there is no comma syntax issue """ try: with open(manifest, encoding='utf-8') as data_file: manifest = json.loads(data_file.read(), object_pairs_hook=check_for_duplicate_keys) except: print_error("[YEP-2.1] Syntax (comma) or encoding issue with manifest.json. Can't check file.") fields = ("name", "id", "packaging_format", "description", "url", "version", "license", "maintainer", "requirements", "multi_instance", "services", "arguments") for field in fields: if field not in manifest: print_warning("[YEP-2.1] \"" + field + "\" field is missing") """ Check values in keys """ if "packaging_format" not in manifest: print_error("[YEP-2.1] \"packaging_format\" key is missing") elif not isinstance(manifest["packaging_format"], int): print_error("[YEP-2.1] \"packaging_format\": value isn't an integer type") elif manifest["packaging_format"] != 1: print_error("[YEP-2.1] \"packaging_format\" field: current format value is '1'") # YEP 1.1 Name is app if "id" in manifest: if not re.match('^[a-z1-9]((_|-)?[a-z1-9])+$', manifest["id"]): print_error("[YEP-1.1] 'id' field '%s' should respect this regex '^[a-z1-9]((_|-)?[a-z1-9])+$'") if "name" in manifest: if len(manifest["name"]) > 22: print_warning( "[YEP-1.1] The 'name' field shouldn't be too long to be able to be with one line in the app list. " "The most current bigger name is actually compound of 22 characters." ) # YEP 1.2 Put the app in a weel known repo if "id" in manifest: app_list_url = "https://raw.githubusercontent.com/YunoHost/apps/master/apps.json" app_list = json.loads(urlopen(app_list_url)['content']) if manifest["id"] not in app_list: print_warning("[YEP-1.2] This app is not registered in our applications list") # YEP 1.3 License def license_mentionned_in_readme(path): readme_path = os.path.join(path, 'README.md') if os.path.isfile(readme_path): return "LICENSE" in open(readme_path).read() return False if "license" in manifest: for license in manifest['license'].replace('&', ',').split(','): code_license = '' + license + '' if license == "nonfree": print_warning("[YEP-1.3] The correct value for non free license in license field is 'non-free' and not 'nonfree'") license = "non-free" if license in ["free", "non-free", "dep-non-free"]: if not license_mentionned_in_readme(self.path): print_warning( "[YEP-1.3] The use of '%s' in license field implies " " to write something about the license in your README.md" % (license) ) if license in ["non-free", "dep-non-free"]: print_warning( "[YEP-1.3] 'non-free' apps can't be officialized. " " Their integration is still being discussed, especially for apps with non-free dependencies" ) elif code_license not in spdx_licenses(): print_warning( "[YEP-1.3] The license '%s' is not registered in https://spdx.org/licenses/ . " "It can be a typo error. If not, you should replace it by 'free' " "or 'non-free' and give some explanations in the README.md." % (license) ) # YEP 1.4 Inform if we continue to maintain the app # YEP 1.5 Update regularly the app status # YEP 1.6 Check regularly the evolution of the upstream # YEP 1.7 - Add an app to the YunoHost-Apps organization if "id" in manifest: repo = "https://github.com/YunoHost-Apps/%s_ynh" % (manifest["id"]) is_not_added_to_org = urlopen(repo)['code'] == 404 brique = "https://github.com/labriqueinternet/%s_ynh" % (manifest["id"]) is_not_added_to_brique = urlopen(brique)['code'] == 404 if is_not_added_to_org and is_not_added_to_brique: print_warning("[YEP-1.7] You should add your app in the YunoHost-Apps organisation.") # YEP 1.8 Publish test request # YEP 1.9 Document app if "description" in manifest: descr = manifest["description"] if isinstance(descr, dict): descr = descr.get("en", None) if descr is None or descr == "" or descr == manifest.get("name", None): print_warning( "[YEP-1.9] You should write a good description of the app, " "at least in english (1 line is enough)." ) if len(descr) > 150: print_warning( "[YEP-1.9] Please use a shorter description (or the rendering on the webadmin / app list will be messy ...). Just describe in consise terms what the app is / does." ) elif "for yunohost" in descr.lower(): print_warning( "[YEP-1.9] The 'description' should explain what the app actually does. " "No need to say that it is 'for YunoHost' - this is a YunoHost app " "so of course we know it is for YunoHost ;-)." ) # TODO test a specific template in README.md # YEP 1.10 Garder un historique de version propre # YEP 1.11 Cancelled # YEP 2.1 if "multi_instance" in manifest and manifest["multi_instance"] != 1 and manifest["multi_instance"] != 0: print_error( "[YEP-2.1] \"multi_instance\" field must be boolean type values 'true' or 'false' and not string type") if "services" in manifest and self.scripts["install"].exists: known_services = ("nginx", "mysql", "uwsgi", "metronome", "php5-fpm", "php7.0-fpm", "php-fpm", "postfix", "dovecot", "rspamd") for service in manifest["services"]: if service not in known_services: if not self.scripts["install"].contains("yunohost service add %s" % service): print_error("[YEP-2.1?] " + service + " service not installed by the install file but present in the manifest") if "install" in manifest["arguments"]: recognized_types = ("domain", "path", "boolean", "app", "password", "user", "string") for argument in manifest["arguments"]["install"]: if "optional" in argument.keys(): if not isinstance(argument["optional"], bool): print_warning("The key 'optional' value for setting %s should be a boolean (true or false)" % argument["name"]) if "type" not in argument.keys(): print_warning( "[YEP-2.1] You should specify the type of the argument '%s'. " "You can use : %s." % (argument["name"], ', '.join(recognized_types)) ) elif argument["type"] not in recognized_types: print_warning( "[YEP-2.1] The type '%s' for argument '%s' is not recognized... " "it probably doesn't behave as you expect ? Choose among those instead : %s" % (argument["type"], argument["name"], ', '.join(recognized_types)) ) if "choices" in argument.keys(): choices = [c.lower() for c in argument["choices"]] if len(choices) == 2: if ("true" in choices and "false" in choices) or ("yes" in choices and "no" in choices): print_warning( "Argument %s : you might want to simply use a boolean-type argument. " "No need to specify the choices list yourself." % argument["name"] ) if argument["name"] == "is_public" and "help" not in argument.keys(): print_warning_not_reliable( "Consider adding an 'help' key for argument 'is_public' " "to explain to the user what it means for *this* app " "to be public or private :\n" ' "help": {\n' ' "en": "Some explanation"\n' ' }') if "url" in manifest and manifest["url"].endswith("_ynh"): print_warning( "'url' is not meant to be the url of the yunohost package, " "but rather the website or repo of the upstream app itself..." ) yunohost_version_req = manifest.get("requirements", {}).get("yunohost", None) if yunohost_version_req: major_version = yunohost_version_req.split()[-1] if major_version.startswith("2"): print_warning( "YunoHost version requirement is still 2.x ... Good job if " "it does still work on Jessie !... But are you really sure " "about that ;) ? be careful that many new helpers you might " "already be playing with are only available on 3.x..." ) class Script(): def __init__(self, app_path, name): self.name = name self.app_path = app_path self.path = app_path + "/scripts/" + name self.exists = file_exists(self.path) if not self.exists: return self.lines = list(self.read_file()) def read_file(self): with open(self.path) as f: lines = f.readlines() # Remove trailing spaces, empty lines and comment lines lines = [line.strip() for line in lines] lines = [line for line in lines if line and not line.startswith('#')] # Merge lines when ending with \ lines = '\n'.join(lines).replace("\\\n", "").split("\n") some_parsing_failed = False for line in lines: try: line = shlex.split(line, True) yield line except Exception as e: if not some_parsing_failed: print("Some lines could not be parsed in script %s. (That's probably not really critical)" % self.name) some_parsing_failed = True print_warning_not_reliable("%s : %s" % (e, line)) def contains(self, command): """ Iterate on lines to check if command is contained in line For instance, "app setting" is contained in "yunohost app setting $app ..." """ return any(command in line for line in [' '.join(line) for line in self.lines]) def containsregex(self, regex): """ Iterate on lines to check if command is contained in line For instance, "app setting" is contained in "yunohost app setting $app ..." """ return any(re.match(regex, line) for line in [' '.join(line) for line in self.lines]) def analyze(self): print_header(self.name.upper() + " SCRIPT") self.check_set_usage() self.check_helper_usage_dependencies() self.check_deprecated_practices() self.check_source_common() def check_set_usage(self): if self.name == "_common.sh": return present = False if self.name in ["backup", "remove"]: present = self.contains("ynh_abort_if_errors") or self.contains("set -eu") else: present = self.contains("ynh_abort_if_errors") if self.name == "remove": # Remove script shouldn't use set -eu or ynh_abort_if_errors if present: print_error( "[YEP-2.4] set -eu or ynh_abort_if_errors is present. " "If there is a crash, it could put yunohost system in " "a broken state. For details, look at " "https://github.com/YunoHost/issues/issues/419" ) elif not present: print_error( "[YEP-2.4] ynh_abort_if_errors is missing. For details, " "look at https://github.com/YunoHost/issues/issues/419" ) def check_helper_usage_dependencies(self): """ Detect usage of ynh_package_* & apt-get * and suggest herlpers ynh_install_app_dependencies and ynh_remove_app_dependencies """ # Skip this in common.sh, sometimes custom not-yet-official helpers need this if self.name == "_common.sh": return if self.contains("ynh_package_install") or self.contains("apt-get install"): print_warning( "You should not use `ynh_package_install` or `apt-get install`, " "use `ynh_install_app_dependencies` instead" ) if self.contains("ynh_package_remove") or self.contains("apt-get remove"): print_warning( "You should not use `ynh_package_remove` or `apt-get remove`, " "use `ynh_remove_app_dependencies` instead" ) def check_deprecated_practices(self): if self.contains("yunohost app setting"): print_error("Do not use 'yunohost app setting' directly. Please use 'ynh_app_setting_(set,get,delete)' instead.") if self.contains("yunohost app checkurl"): print_error("'yunohost app checkurl' is obsolete!!! Please use 'ynh_webpath_register' instead.") if self.contains("yunohost app checkport"): print_error("'yunohost app checkport' is obsolete!!! Please use 'ynh_find_port' instead.") if self.contains("yunohost app initdb"): print_error("'yunohost app initdb' is obsolete!!! Please use 'ynh_mysql_setup_db' instead.") if self.contains("exit"): print_warning("'exit' command shouldn't be used. Please use 'ynh_die' instead.") if self.contains("yunohost service regen-conf"): print_warning("'yunohost tools regen-conf' has been replaced by 'yunohost tools regen-conf'.") # Dirty hack to check only the 10 last lines for ssowatconf # (the "bad" practice being using this at the very end of the script, but some apps legitimately need this in the middle of the script) oldlines = list(self.lines) self.lines = self.lines[-10:] if self.contains("yunohost app ssowatconf"): print_warning("You probably don't need to run 'yunohost app ssowatconf' in the app script. It's supposed to be ran automatically after the script.") self.lines = oldlines if self.contains("rm -rf"): print_error("[YEP-2.12] You should avoid using 'rm -rf', please use 'ynh_secure_remove' instead") if self.contains("sed -i"): print_warning("[YEP-2.12] You should avoid using 'sed -i', please use 'ynh_replace_string' instead") if self.containsregex(r"sudo \w"): # \w is here to not match sudo -u, legit use because ynh_exec_as not official yet... print_warning( "[YEP-2.12] You should not need to use 'sudo', the script is being run as root. " "(If you need to run a command using a specific user, use 'ynh_exec_as')" ) if self.contains("dd if=/dev/urandom") or self.contains("openssl rand"): print_warning( "Instead of 'dd if=/dev/urandom' or 'openssl rand', " "you might want to use ynh_string_random" ) if self.contains("systemctl restart nginx") or self.contains("service nginx restart"): print_error( "Restarting nginx is quite dangerous (especially for web installs) " "and should be avoided at all cost. Use 'reload' instead." ) if self.name == "install" and not self.contains("ynh_print_info") and not self.contains("ynh_script_progression"): print_warning( "Please add a few messages for the user, to explain what is going on " "(in friendly, not-too-technical terms) during the installation. " "You can use 'ynh_print_info' or 'ynh_script_progression' for this." ) if self.name == "install": if self.contains("/etc/apt/sources.list") \ or (os.path.exists(self.app_path + "/scripts/_common.sh") and "/etc/apt/sources.list" in open(self.app_path+"/scripts/_common.sh").read() and "ynh_add_repo" not in open(self.app_path+"/scripts/_common.sh").read()): print_error( "[YEP-3.7] Manually messing with apt's sources.lists is strongly discouraged " "and should be avoided. Please consider alternatives like using a .deb directly " "or using experimental helpers (c.f. " "https://github.com/YunoHost-Apps/Experimental_helpers/tree/master/ynh_add_extra_apt_repos )" ) def check_source_common(self): if self.name in ["backup", "restore"]: if self.contains("source _common.sh") or self.contains("source ./_common.sh"): print_warning("In the context of backup and restore script, you should load _common.sh with \"source ../settings/scripts/_common.sh\"") def main(): if len(sys.argv) != 2: print("Give one app package path.") exit() app_path = sys.argv[1] header(app_path) App(app_path).analyze() if error_count > 0: sys.exit(1) elif warning_count > 3: print("Still some warnings to be fixed :s") elif warning_count > 0: print("Only %s warning remaining! You can do it!" % warning_count) else: print_happy("Not even a warning! Congratz and thank you for keeping that package up to date with good practices !") if __name__ == '__main__': main()