#!/usr/env/python3 import sys import os import glob import datetime import subprocess tree = { "sources": { "title": "Sources", "notes": "This is coupled to the 'sources' resource in the manifest.toml", "subsections": ["sources"], }, "tech": { "title": "App technologies", "notes": "These allow to install specific version of the technology required to run some apps", "subsections": ["nodejs", "ruby", "go", "composer"], }, "db": { "title": "Databases", "notes": "This is coupled to the 'database' resource in the manifest.toml - at least for mysql/postgresql. Mongodb/redis may have better integration in the future.", "subsections": ["mysql", "postgresql", "mongodb", "redis"], }, "conf": { "title": "Configurations / templating", "subsections": [ "templating", "nginx", "php", "systemd", "fail2ban", "logrotate", ], }, "misc": { "title": "Misc tools", "subsections": [ "utils", "setting", "string", "backup", "logging", "multimedia", ], }, "meh": { "title": "Deprecated or handled by the core / app resources since v2", "subsections": ["permission", "apt", "systemuser"], }, } def get_current_commit(): p = subprocess.Popen( "git rev-parse --verify HEAD", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = p.communicate() current_commit = stdout.strip().decode("utf-8") return current_commit def render(tree, helpers_version): from jinja2 import Template from ansi2html import Ansi2HTMLConverter from ansi2html.style import get_styles conv = Ansi2HTMLConverter() shell_css = "\n".join(map(str, get_styles(conv.dark_bg))) def shell_to_html(shell): return conv.convert(shell, False) template = open("helper_doc_template.md", "r").read() t = Template(template) t.globals["now"] = datetime.datetime.utcnow result = t.render( tree=tree, date=datetime.datetime.now().strftime("%d/%m/%Y"), version=open("../debian/changelog").readlines()[0].split()[1].strip("()"), helpers_version=helpers_version, current_commit=get_current_commit(), convert=shell_to_html, shell_css=shell_css, ) open(f"helpers.v{helpers_version}.md", "w").write(result) ############################################################################## class Parser: def __init__(self, filename): self.filename = filename self.file = open(filename, "r").readlines() self.blocks = None def parse_blocks(self): self.blocks = [] current_reading = "void" current_block = {"name": None, "line": -1, "comments": [], "code": []} for i, line in enumerate(self.file): if line.startswith("#!/bin/bash"): continue line = line.rstrip().replace("\t", " ") if current_reading == "void": if is_global_comment(line): # We start a new comment bloc current_reading = "comments" assert line.startswith("# ") or line == "#", malformed_error(i) current_block["comments"].append(line[2:]) else: pass # assert line == "", malformed_error(i) continue elif current_reading == "comments": if is_global_comment(line): # We're still in a comment bloc assert line.startswith("# ") or line == "#", malformed_error(i) current_block["comments"].append(line[2:]) elif line.strip() == "" or line.startswith("_ynh"): # Well eh that was not an actual helper definition ... start over ? current_reading = "void" current_block = { "name": None, "line": -1, "comments": [], "code": [], } elif not (line.endswith("{") or line.endswith("()")): # Well we're not actually entering a function yet eh # (c.f. global vars) pass else: # We're getting out of a comment bloc, we should find # the name of the function assert len(line.split()) >= 1, "Malformed line {} in {}".format( i, self.filename, ) current_block["line"] = i current_block["name"] = line.split()[0].strip("(){") # Then we expect to read the function current_reading = "code" elif current_reading == "code": if line == "}": # We're getting out of the function current_reading = "void" # Then we keep this bloc and start a new one # (we ignore helpers containing [internal] ...) if ( "[packagingv1]" not in current_block["comments"] and not any( line.startswith("[internal]") for line in current_block["comments"] ) and not current_block["name"].startswith("_") ): self.blocks.append(current_block) current_block = { "name": None, "line": -1, "comments": [], "code": [], } else: current_block["code"].append(line) continue def parse_block(self, b): b["brief"] = "" b["details"] = "" b["usage"] = "" b["args"] = [] b["ret"] = "" subblocks = "\n".join(b["comments"]).split("\n\n") for i, subblock in enumerate(subblocks): subblock = subblock.strip() if i == 0: b["brief"] = subblock continue elif subblock.startswith("example:"): b["example"] = " ".join(subblock.split()[1:]) continue elif subblock.startswith("examples:"): b["examples"] = subblock.split("\n")[1:] continue elif subblock.startswith("usage"): for line in subblock.split("\n"): if line.startswith("| arg"): linesplit = line.split() argname = linesplit[2] # Detect that there's a long argument version (-f, --foo - Some description) if argname.endswith(",") and linesplit[3].startswith("--"): argname = argname.strip(",") arglongname = linesplit[3] argdescr = " ".join(linesplit[5:]) b["args"].append((argname, arglongname, argdescr)) else: argdescr = " ".join(linesplit[4:]) b["args"].append((argname, argdescr)) elif line.startswith("| ret"): b["ret"] = " ".join(line.split()[2:]) else: if line.startswith("usage"): line = " ".join(line.split()[1:]) b["usage"] += line + "\n" continue elif subblock.startswith("| arg"): for line in subblock.split("\n"): if line.startswith("| arg"): linesplit = line.split() argname = linesplit[2] # Detect that there's a long argument version (-f, --foo - Some description) if argname.endswith(",") and linesplit[3].startswith("--"): argname = argname.strip(",") arglongname = linesplit[3] argdescr = " ".join(linesplit[5:]) b["args"].append((argname, arglongname, argdescr)) else: argdescr = " ".join(linesplit[4:]) b["args"].append((argname, argdescr)) continue else: b["details"] += subblock + "\n\n" b["usage"] = b["usage"].strip() def is_global_comment(line): return line.startswith("#") def malformed_error(line_number): return "Malformed file line {} ?".format(line_number) def main(): if len(sys.argv) == 1: print("This script needs the helper version (1, 2, 2.1) as an argument") sys.exit(1) version = sys.argv[1] for section in tree.values(): section["helpers"] = {} for subsection in section["subsections"]: print(f"Parsing {subsection} ...") helper_file = f"../helpers/helpers.v{version}.d/{subsection}" assert os.path.isfile(helper_file), f"Uhoh, {file} doesn't exists?" p = Parser(helper_file) p.parse_blocks() for b in p.blocks: p.parse_block(b) section["helpers"][subsection] = p.blocks render(tree, version) main()