Merge pull request #2175 from YunoHost/simplify_list_builder

Add 'added_date' and 'killed_date' to catalog files.

.github/workflows/added_dates.yml

@@ -0,0 +1,32 @@
+name: Add Added dates to the catalog files
+
+on:
+  push:
+  pull_request_target:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - name: Install toml python lib
+      run: |
+        pip3 install toml tomlkit gitpython
+    - name: Add added dates to catalog files
+      run: |
+        ./tools/save_added_date.py
+
+    - uses: uncenter/setup-taplo@v1
+    - run: taplo fmt
+
+    - uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: "Automatically add dates to the catalog files"
+        file_pattern: 'apps.toml wishlist.toml graveyard.toml'

graveyard.toml

@@ -1,259 +1,371 @@
 #:schema https://github.com/YunoHost/apps/raw/master/schemas/graveyard.toml.schema.json
 
 [anfora]
+added_date = 1554588215 # 2019/04/07
 category = "social_media"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "pictures" ]
 url = "https://github.com/YunoHost-Apps/anfora_ynh"
 
 [bibliogram]
+added_date = 1597045792 # 2020/08/10
 category = "social_media"
+killed_date = 1703600553 # 2023/12/26
 potential_alternative_to = [ "Instagram" ]
 subtags = [ "pictures" ]
 url = "https://github.com/YunoHost-Apps/bibliogram_ynh"
 
 [democracyos]
+added_date = 1554588215 # 2019/04/07
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "forum" ]
 url = "https://github.com/YunoHost-Apps/democracyos_ynh"
 
 [dockerui]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/dockerui_ynh"
 
 [dynette]
+added_date = 1554588215 # 2019/04/07
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/dynette_ynh"
 
 [ecko]
+added_date = 1636827596 # 2021/11/13
 category = "social_media"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "microblogging" ]
 url = "https://github.com/YunoHost-Apps/ecko_ynh"
 
 [fallback]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "backup" ]
 url = "https://github.com/YunoHost-Apps/fallback_ynh"
 
 [flask]
+added_date = 1554588215 # 2019/04/07
 category = "dev"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "skeleton" ]
 url = "https://github.com/YunoHost-Apps/flask_ynh"
 
 [flusio]
+added_date = 1660509055 # 2022/08/14
 category = "reading"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "rssreader" ]
 url = "https://github.com/YunoHost-Apps/flusio_ynh"
 
 [foodsoft]
+added_date = 1564517377 # 2019/07/30
 category = "productivity_and_management"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "business_and_ngos" ]
 url = "https://github.com/YunoHost-Apps/foodsoft_ynh"
 
 [framaestro]
+added_date = 1554588215 # 2019/04/07
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "meeting" ]
 url = "https://github.com/YunoHost-Apps/framaestro_ynh"
 
 [framaestro_hub]
+added_date = 1554588215 # 2019/04/07
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "meeting" ]
 url = "https://github.com/YunoHost-Apps/framaestro_hub_ynh"
 
 [freeboard]
+added_date = 1554588215 # 2019/04/07
 category = "iot"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/freeboard_ynh"
 
 [freepbx]
+added_date = 1590399842 # 2020/05/25
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/freepbx_ynh"
 
 [ftp_webapp]
+added_date = 1554588215 # 2019/04/07
 category = "small_utilities"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/ftp_support_webapp_ynh"
 
 [ftssolr]
+added_date = 1590399842 # 2020/05/25
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/ftssolr_ynh"
 
 [gekko]
+added_date = 1590399842 # 2020/05/25
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/gekko_ynh"
 
 [gitrepositories]
+added_date = 1554588215 # 2019/04/07
 category = "dev"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "forge" ]
 url = "https://github.com/YunoHost-Apps/gitrepositories_ynh"
 
 [gnusocial]
+added_date = 1554588215 # 2019/04/07
 category = "social_media"
+killed_date = 1703600553 # 2023/12/26
 potential_alternative_to = [ "X" ]
 subtags = [ "microblogging" ]
 url = "https://github.com/YunoHost-Apps/gnusocial_ynh"
 
 [gogswebhost]
+added_date = 1590399842 # 2020/05/25
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/gogs_webhost_ynh"
 
 [internetarchive]
+added_date = 1569663577 # 2019/09/28
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/internetarchive_ynh"
 
 [jappix_mini]
+added_date = 1554588215 # 2019/04/07
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "chat" ]
 url = "https://github.com/YunoHost-Apps/jappix_mini_ynh"
 
 [lbcalerte]
+added_date = 1554588215 # 2019/04/07
 category = "small_utilities"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/lbcalerte_ynh"
 
 [lektor]
+added_date = 1554588215 # 2019/04/07
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/lektor_ynh"
 
 [mailman]
+added_date = 1554588215 # 2019/04/07
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 potential_alternative_to = [ "Google Groups" ]
 subtags = [ "email" ]
 url = "https://github.com/yunohost-apps/mailman_ynh"
 
 [mediadrop]
+added_date = 1554588215 # 2019/04/07
 category = "multimedia"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "mediacenter" ]
 url = "https://github.com/YunoHost-Apps/mediadrop_ynh"
 
 [menu]
+added_date = 1554588215 # 2019/04/07
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/menu_ynh"
 
 [modernpaste]
+added_date = 1554588215 # 2019/04/07
 category = "small_utilities"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "pastebin" ]
 url = "https://github.com/YunoHost-Apps/modernpaste_ynh"
 
 [monit]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "monitoring" ]
 url = "https://github.com/YunoHost-Apps/monit_ynh"
 
 [multi_webapp]
+added_date = 1554588215 # 2019/04/07
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/multi_webapp_ynh"
 
 [munin]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "monitoring" ]
 url = "https://github.com/YunoHost-Apps/munin_ynh"
 
 [nexusoss]
+added_date = 1554588215 # 2019/04/07
 category = "dev"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/nexusoss_ynh"
 
 [ntopng]
+added_date = 1590399842 # 2020/05/25
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/ntopng_ynh"
 
 [osmw]
+added_date = 1554588215 # 2019/04/07
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/osmw_ynh"
 
 [peachpub]
+added_date = 1655358480 # 2022/06/16
 category = "communication"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/peachpub_ynh"
 
 [piratebox]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "network" ]
 url = "https://github.com/labriqueinternet/piratebox_ynh"
 
 [plonecms]
+added_date = 1554588215 # 2019/04/07
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/plonecms_ynh"
 
 [portainer]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/portainer_ynh"
 
 [reel2bits]
+added_date = 1554588215 # 2019/04/07
 category = "social_media"
+killed_date = 1703600553 # 2023/12/26
 potential_alternative_to = [ "Soundcloud" ]
 subtags = [ "music" ]
 url = "https://github.com/YunoHost-Apps/reel2bits_ynh"
 
 [remotestorage]
+added_date = 1554588215 # 2019/04/07
 category = "small_utilities"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/remotestorage_ynh"
 
 [roadiz]
+added_date = 1554588215 # 2019/04/07
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/roadiz_ynh"
 
 [shsd]
+added_date = 1554588215 # 2019/04/07
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "monitoring" ]
 url = "https://github.com/YunoHost-Apps/shsd_ynh"
 
 [sickbeard]
+added_date = 1554588215 # 2019/04/07
 category = "multimedia"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "download" ]
 url = "https://github.com/YunoHost-Apps/sickbeard_ynh"
 
 [sickrage]
+added_date = 1554588215 # 2019/04/07
 category = "multimedia"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "download" ]
 url = "https://github.com/YunoHost-Apps/sickrage_ynh"
 
 [sonerezh]
+added_date = 1554588215 # 2019/04/07
 category = "multimedia"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "music" ]
 url = "https://github.com/YunoHost-Apps/sonerezh_ynh"
 
 [staticwebapp]
+added_date = 1554588215 # 2019/04/07
 category = "publishing"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/staticwebapp_ynh"
 
 [subscribe]
+added_date = 1590399842 # 2020/05/25
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/subscribe_ynh"
 
 [tagspaces]
+added_date = 1554588215 # 2019/04/07
 category = "synchronization"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/tagspaces_ynh"
 
 [telegram_chatbot]
+added_date = 1554588215 # 2019/04/07
 category = "dev"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/telegram_chatbot_ynh"
 
 [tes3mp]
+added_date = 1554588215 # 2019/04/07
 category = "games"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/tes3mp_ynh"
 
 [transpay]
+added_date = 1571446958 # 2019/10/19
 category = "productivity_and_management"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/transpay_ynh"
 
 [unbound]
+added_date = 1590399842 # 2020/05/25
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/unbound_ynh"
 
 [vpnserver]
+added_date = 1564314757 # 2019/07/28
 category = "system_tools"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "network" ]
 url = "https://github.com/YunoHost-Apps/vpnserver_ynh"
 
 [wildfly]
+added_date = 1554588215 # 2019/04/07
 category = "dev"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/wildfly_ynh"
 
 [youtube-dl-webui]
+added_date = 1554588215 # 2019/04/07
 category = "multimedia"
+killed_date = 1703600553 # 2023/12/26
 subtags = [ "download" ]
 url = "https://github.com/YunoHost-Apps/youtube-dl-webui_ynh"
 
 [yunofav]
+added_date = 1554588215 # 2019/04/07
 category = "wat"
+killed_date = 1703600553 # 2023/12/26
 url = "https://github.com/YunoHost-Apps/yunofav_ynh"

schemas/apps.toml.schema.json

@@ -56,6 +56,12 @@
                 },
                 "branch": {
                     "type": "string"
+                },
+                "added_date": {
+                    "type": "integer"
+                },
+                "deprecated_date": {
+                    "type": "integer"
                 }
             }
         }

schemas/graveyard.toml.schema.json

@@ -41,8 +41,17 @@
                         "type": "string"
                     },
                     "additionalItems": false
+                },
+                "added_date": {
+                    "type": "integer"
+                },
+                "deprecated_date": {
+                    "type": "integer"
+                },
+                "killed_date": {
+                    "type": "integer"
                 }
             }
         }
     }
-}
+}

schemas/wishlist.toml.schema.json

@@ -32,8 +32,11 @@
                 "draft": {
                     "type": "string",
                     "format": "url"
+                },
+                "added_date": {
+                    "type": "integer"
                 }
             }
         }
     }
-}
+}

taplo.toml

@@ -1,6 +1,7 @@
 include = [ "antifeatures.toml", "apps.toml", "categories.toml", "graveyard.toml", "taplo.toml" ]
 
 [formatting]
+align_comments = false
 array_auto_expand = false
 compact_arrays = false
 indent_tables = true

tools/list_builder.py

@@ -193,42 +193,8 @@ def build_app_dict(app, infos):
 
     repo = Repo(this_app_cache)
 
-    commits_in_apps_json = (
-        Repo(REPO_APPS_ROOT)
-        .git.log(
-            "-S",
-            f'"{app}"',
-            "--first-parent",
-            "--reverse",
-            "--date=unix",
-            "--format=%cd",
-            "--",
-            "apps.json",
-        )
-        .split("\n")
-    )
-    if len(commits_in_apps_json) > 1:
-        first_commit = commits_in_apps_json[0]
-    else:
-        commits_in_apps_toml = (
-            Repo(REPO_APPS_ROOT)
-            .git.log(
-                "-S",
-                f"[{app}]",
-                "--first-parent",
-                "--reverse",
-                "--date=unix",
-                "--format=%cd",
-                "--",
-                "apps.json",
-                "apps.toml",
-            )
-            .split("\n")
-        )
-        first_commit = commits_in_apps_toml[0]
-
-    # Assume the first entry we get (= the oldest) is the time the app was added
-    infos["added_in_catalog"] = int(first_commit)
+    # If added_date is not present, we are in a github action of the PR that adds it... so default to a bad value.
+    infos["added_in_catalog"] = infos.get("added_date", 0)
     # int(commit_timestamps_for_this_app_in_catalog.split("\n")[0])
 
     infos["branch"] = infos.get("branch", "master")

tools/save_added_date.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+import tomlkit
+import json
+from datetime import datetime
+from git import Repo, Commit
+from pathlib import Path
+import logging
+from typing import TYPE_CHECKING, Callable
+
+if TYPE_CHECKING:
+    REPO_APPS_ROOT = Path()
+else:
+    from appslib.utils import REPO_APPS_ROOT
+
+
+def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None:
+    repo = Repo(repo_path)
+
+    # Start with whole repo
+    first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0)
+    repo.git.bisect("reset")
+    repo.git.bisect("start", "--no-checkout", "HEAD", first_commit)
+
+    while True:
+        try:
+            status = "bad" if is_newer(repo.commit("BISECT_HEAD")) else "good"
+        except Exception:
+            status = "skip"
+        result_string = repo.git.bisect(status)
+        if "is the first bad commit" in result_string.splitlines()[0]:
+            return repo.commit(result_string.splitlines()[0].split(" ", 1)[0])
+
+
+def get_app_info(commit: Commit, filebase: str, name: str) -> dict | None:
+    data = None
+    try:
+        filestream = commit.tree.join(f"{filebase}.toml")
+        filedata = filestream.data_stream.read().decode("utf-8")
+        dictdata = tomlkit.loads(filedata)
+        data = dictdata[name]
+    except KeyError:
+        pass
+    try:
+        filestream = commit.tree.join(f"{filebase}.json")
+        filedata = filestream.data_stream.read().decode("utf-8")
+        dictdata = json.loads(filedata)
+        data = dictdata[name]
+    except KeyError:
+        pass
+
+    assert isinstance(data, dict) or data is None
+    return data
+
+
+def app_is_present(commit: Commit, name: str) -> bool:
+    info = get_app_info(commit, "apps", name)
+    # if info is None:
+    #     info = get_app_info(commit, "graveyard", name)
+    return info is not None
+
+
+def app_is_deprecated(commit: Commit, name: str) -> bool:
+    info = get_app_info(commit, "apps", name)
+    if info is None:
+        return False
+
+    antifeatures = info.get("antifeatures", [])
+    return "deprecated-software" in antifeatures
+
+
+def date_added(name: str) -> int | None:
+    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name))
+    print(result)
+    return None if result is None else result.committed_date
+
+
+def date_deprecated(name: str) -> int | None:
+    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name))
+    print(result)
+    return None if result is None else result.committed_date
+
+
+def add_deprecation_dates(file: Path) -> None:
+    key = "deprecated_date"
+    document = tomlkit.load(file.open("r", encoding="utf-8"))
+    for app, info in document.items():
+        if key in info.keys():
+            continue
+        if "deprecated-software" not in info.get("antifeatures", []):
+            continue
+        date = date_deprecated(app)
+        if date is None:
+            continue
+        info[key] = date
+        info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
+        info[key].trivia.comment_ws = "  "
+    tomlkit.dump(document, file.open("w"))
+
+
+def date_added_to(match: str, file: Path) -> int | None:
+    commits = Repo(REPO_APPS_ROOT).git.log(
+        "-S", match, "--first-parent", "--reverse",
+        "--date=unix", "--format=%cd", "--", str(file)).splitlines()
+
+    if not commits:
+        return None
+    first_commit = commits[0]
+    return int(first_commit)
+
+
+def add_apparition_dates(file: Path, key: str) -> None:
+    document = tomlkit.load(file.open("r", encoding="utf-8"))
+    for app, info in document.items():
+        if key in info.keys():
+            continue
+        date = date_added_to(f"[{app}]", file)
+        assert date is not None
+        info[key] = date
+        info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
+        info[key].trivia.comment_ws = "  "
+    tomlkit.dump(document, file.open("w"))
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.DEBUG)
+
+    add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date")
+    add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date")
+    add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date")
+
+    add_deprecation_dates(REPO_APPS_ROOT/"apps.toml")
+    add_deprecation_dates(REPO_APPS_ROOT/"graveyard.toml")
+
+
+if __name__ == "__main__":
+    main()