helpers: rework helper doc now that we have multiple versions of helpers in parallel + improve structure (group helper file in categories)

.gitlab/ci/doc.gitlab-ci.yml

@@ -12,10 +12,12 @@ generate-helpers-doc:
     - git config --global user.name "$GITHUB_USER"
   script:
     - cd doc
-    - python3 generate_helper_doc.py
+    - python3 generate_helper_doc.py 2
+    - python3 generate_helper_doc.py 2.1
     - python3 generate_resource_doc.py > resources.md
     - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo
-    - cp helpers.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md
+    - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md
+    - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers_v2.1.md
     - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md
     - cd doc_repo
     # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ?

doc/generate_helper_doc.py

@@ -1,10 +1,40 @@
 #!/usr/env/python3
 
+import sys
 import os
 import glob
 import datetime
 import subprocess
 
+tree = {
+    "sources": {
+        "title": "Sources",
+        "notes": "This is coupled to the 'sources' resource in the manifest.toml",
+        "subsections": ["sources"],
+    },
+    "tech": {
+        "title": "App technologies",
+        "notes": "These allow to install specific version of the technology required to run some apps",
+        "subsections": ["nodejs", "ruby", "go", "composer"],
+    },
+    "db": {
+        "title": "Databases",
+        "notes": "This is coupled to the 'database' resource in the manifest.toml - at least for mysql/postgresql. Mongodb/redis may have better integration in the future.",
+        "subsections": ["mysql", "postgresql", "mongodb", "redis"],
+     },
+    "conf": {
+        "title": "Configurations / templating",
+        "subsections": ["templating", "nginx", "php", "systemd", "fail2ban", "logrotate"],
+    },
+    "misc": {
+        "title": "Misc tools",
+        "subsections": ["utils", "setting", "string", "backup", "logging", "multimedia"],
+    },
+    "meh": {
+        "title": "Deprecated or handled by the core / app resources since v2",
+        "subsections": ["permission", "apt", "systemuser"],
+    },
+}
 
 def get_current_commit():
     p = subprocess.Popen(
@@ -19,14 +49,7 @@ def get_current_commit():
     return current_commit
 
 
-def render(helpers):
+def render(tree, helpers_version):
-    current_commit = get_current_commit()
-
-    data = {
-        "helpers": helpers,
-        "date": datetime.datetime.now().strftime("%d/%m/%Y"),
-        "version": open("../debian/changelog").readlines()[0].split()[1].strip("()"),
-    }
 
     from jinja2 import Template
     from ansi2html import Ansi2HTMLConverter
@@ -42,12 +65,15 @@ def render(helpers):
     t = Template(template)
     t.globals["now"] = datetime.datetime.utcnow
     result = t.render(
-        current_commit=current_commit,
+        tree=tree,
-        data=data,
+        date=datetime.datetime.now().strftime("%d/%m/%Y"),
+        version=open("../debian/changelog").readlines()[0].split()[1].strip("()"),
+        helpers_version=helpers_version,
+        current_commit=get_current_commit(),
         convert=shell_to_html,
         shell_css=shell_css,
     )
-    open("helpers.md", "w").write(result)
+    open(f"helpers.v{helpers_version}.md", "w").write(result)
 
 
 ##############################################################################
@@ -87,7 +113,7 @@ class Parser:
                     # We're still in a comment bloc
                     assert line.startswith("# ") or line == "#", malformed_error(i)
                     current_block["comments"].append(line[2:])
-                elif line.strip() == "":
+                elif line.strip() == "" or line.startswith("_ynh"):
                     # Well eh that was not an actual helper definition ... start over ?
                     current_reading = "void"
                     current_block = {
@@ -121,7 +147,8 @@ class Parser:
                     # (we ignore helpers containing [internal] ...)
                     if (
                         "[packagingv1]" not in current_block["comments"]
-                        and "[internal]" not in current_block["comments"]
+                        and not any(line.startswith("[internal]") for line in current_block["comments"])
+                        and not current_block["name"].startswith("_")
                     ):
                         self.blocks.append(current_block)
                     current_block = {
@@ -212,23 +239,27 @@ def malformed_error(line_number):
 
 
 def main():
-    helper_files = sorted(glob.glob("../helpers/*"))
-    helpers = []
 
-    for helper_file in helper_files:
+    if len(sys.argv) == 1:
-        if not os.path.isfile(helper_file):
+        print("This script needs the helper version (1, 2, 2.1) as an argument")
-            continue
+        sys.exit(1)
 
-        category_name = os.path.basename(helper_file)
+    version = sys.argv[1]
-        print("Parsing %s ..." % category_name)
-        p = Parser(helper_file)
-        p.parse_blocks()
-        for b in p.blocks:
-            p.parse_block(b)
 
-        helpers.append((category_name, p.blocks))
+    for section in tree.values():
+        section["helpers"] = {}
+        for subsection in section["subsections"]:
+            print(f"Parsing {subsection} ...")
+            helper_file = f"../helpers/helpers.v{version}.d/{subsection}"
+            assert os.path.isfile(helper_file), f"Uhoh, {file} doesn't exists?"
+            p = Parser(helper_file)
+            p.parse_blocks()
+            for b in p.blocks:
+                p.parse_block(b)
 
-    render(helpers)
+            section["helpers"][subsection] = p.blocks
+
+    render(tree, version)
 
 
 main()

doc/helper_doc_template.md

@@ -1,19 +1,27 @@
 ---
-title: App helpers
+title: App helpers (v{{ helpers_version }})
 template: docs
 taxonomy:
     category: docs
 routes:
-  default: '/packaging_apps_helpers'
+  default: '/packaging_apps_helpers{% if helpers_version not in ["1", "2"] %}_v{{ helpers_version }}{% endif %}'
 ---
 
-Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{data.date}} (YunoHost version {{data.version}})
+Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/doc/generate_helper_doc.py) on {{date}} (YunoHost version {{version}})
 
-{% for category, helpers in data.helpers %}
+
-## {{ category.upper() }}
+{% for section_id, section in tree.items() %}
-{% for h in helpers %}
+## {{ section["title"].title() }}
-### {{ h.name }}
+
-[details summary="<i>{{ h.brief }}</i>" class="helper-card-subtitle text-muted"]
+{% if section['notes'] %}<p>{{ section['notes'] }}</p>{% endif %}
+
+    {% for subsection, helpers in section["helpers"].items() %}
+
+### {{ subsection.upper() }}
+        {% for h in helpers %}
+#### {{ h.name }}
+
+<details> <summary><i>{{ h.brief }}</i></summary>
 
 **Usage**: `{{ h.usage }}`
 {%- if h.args %}
@@ -51,9 +59,9 @@ Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{{
 **Details**:  
 {{ h.details }}
 {%- endif %}
-[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/{{ category }}#L{{ h.line + 1 }})
+[Dude, show me the code!](https://github.com/YunoHost/yunohost/blob/{{ current_commit }}/helpers/helpers.v{{ helpers_version if helpers_version != "2" else "1" }}.d/{{ subsection }}#L{{ h.line + 1 }})
-[/details]
+</details>
-
+        {% endfor %}
 ---
-{% endfor %}
+    {% endfor %}
 {% endfor %}