From 2f3fcd24e23802b8c337e8d1273060b691473967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 23:28:54 +0100 Subject: [PATCH 01/10] Rework list_builder.py FAAAAAASTEEEEEEER --- tools/app_caches.py | 2 +- tools/appslib/xmpplogger.py | 33 +++++ tools/list_builder.py | 281 ++++++++++++------------------------ 3 files changed, 128 insertions(+), 188 deletions(-) create mode 100644 tools/appslib/xmpplogger.py diff --git a/tools/app_caches.py b/tools/app_caches.py index f3e35fd3..180c4dfa 100755 --- a/tools/app_caches.py +++ b/tools/app_caches.py @@ -76,7 +76,7 @@ def __app_cache_clone_or_update_mapped(data): def apps_cache_update_all(apps: dict[str, dict[str, Any]], parallel: int = 8) -> None: with Pool(processes=parallel) as pool: tasks = pool.imap_unordered(__app_cache_clone_or_update_mapped, apps.items()) - for _ in tqdm.tqdm(tasks, total=len(apps.keys())): + for _ in tqdm.tqdm(tasks, total=len(apps.keys()), ascii=" ·#"): pass diff --git a/tools/appslib/xmpplogger.py b/tools/appslib/xmpplogger.py new file mode 100644 index 00000000..69c48148 --- /dev/null +++ b/tools/appslib/xmpplogger.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import subprocess +from shutil import which +import logging +import logging.handlers + + +class XmppLogHandler(logging.Handler): + def __init__(self): + logging.Handler.__init__(self) + self.is_logging = False + + def emit(self, record): + if which("sendxmpppy") is None: + return + + msg = f"[Applist builder error] {record.msg}" + subprocess.call(["sendxmpppy", msg], stdout=subprocess.DEVNULL) + + @classmethod + def add(cls, level=logging.ERROR): + if not logging.getLogger().handlers: + logging.basicConfig() + + # create handler + handler = cls() + handler.setLevel(level) + # add the handler + logging.getLogger().handlers.append(handler) + + +XmppLogHandler.add(logging.ERROR) diff --git a/tools/list_builder.py b/tools/list_builder.py index 1994f016..1bb947a0 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -3,27 +3,34 @@ import copy import json import os -import re import subprocess -import sys -import time -from collections import OrderedDict +import multiprocessing from pathlib import Path -from shutil import which -from typing import Any, Generator, TextIO +import time +import shutil +from collections import OrderedDict +import tqdm +import logging import toml from git import Repo +from app_caches import apps_cache_update_all, app_cache_folder # pylint: disable=import-error from packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ convert_v1_manifest_to_v2_for_catalog # pylint: disable=import-error +from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error + get_catalog, git_repo_age) + +# Automatically enables error-to-xmpp +import appslib.xmpplogger # pylint: disable=import-error + + now = time.time() -REPO_APPS_PATH = Path(__file__).parent.parent # Load categories and reformat the structure to have a list with an "id" key -categories = toml.load((REPO_APPS_PATH / "categories.toml").open("r", encoding="utf-8")) +categories = toml.load((REPO_APPS_ROOT / "categories.toml").open("r", encoding="utf-8")) for category_id, infos in categories.items(): infos["id"] = category_id for subtag_id, subtag_infos in infos.get("subtags", {}).items(): @@ -33,13 +40,13 @@ for category_id, infos in categories.items(): categories = list(categories.values()) # (Same for antifeatures) -antifeatures = toml.load((REPO_APPS_PATH / "antifeatures.toml").open("r", encoding="utf-8")) +antifeatures = toml.load((REPO_APPS_ROOT / "antifeatures.toml").open("r", encoding="utf-8")) for antifeature_id, infos in antifeatures.items(): infos["id"] = antifeature_id antifeatures = list(antifeatures.values()) # Load the app catalog and filter out the non-working ones -catalog = toml.load((REPO_APPS_PATH / "apps.toml").open("r", encoding="utf-8")) +catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) catalog = { app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" } @@ -47,165 +54,55 @@ catalog = { my_env = os.environ.copy() my_env["GIT_TERMINAL_PROMPT"] = "0" -(REPO_APPS_PATH / ".apps_cache").mkdir(exist_ok=True) -(REPO_APPS_PATH / "builds").mkdir(exist_ok=True) - - -def error(msg: str) -> None: - msg = "[Applist builder error] " + msg - if which("sendxmpppy") is not None: - subprocess.call(["sendxmpppy", msg], stdout=open(os.devnull, "wb")) - print(msg + "\n") - - -# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 -def progressbar(it: list[Any], prefix: str = "", size: int = 60, file: TextIO = sys.stdout - ) -> Generator[Any, None, None]: - count = len(it) - - def show(j, name=""): - name += " " - x = int(size * j / count) - file.write( - "%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name) - ) - file.flush() - - show(0) - for i, item in enumerate(it): - yield item - show(i + 1, item[0]) - file.write("\n") - file.flush() - - -################################### -# App git clones cache management # -################################### - - -def app_cache_folder(app: str) -> Path: - return REPO_APPS_PATH / ".apps_cache" / app - - -def refresh_all_caches() -> None: - for app, infos in progressbar(sorted(catalog.items()), "Updating git clones: ", 40): - app = app.lower() - if not app_cache_folder(app).exists(): - try: - init_cache(app, infos) - except Exception as e: - error("Failed to init cache for %s" % app) - else: - try: - refresh_cache(app, infos) - except Exception as e: - error("Failed to not refresh cache for %s: %s" % (app, e)) - raise e - - -def init_cache(app: str, infos: dict[str, str]) -> None: - git_depths = { - "notworking": 5, - "inprogress": 20, - "default": 40, - } - - Repo.clone_from( - infos["url"], - to_path=app_cache_folder(app), - depth=git_depths.get(infos["state"], git_depths["default"]), - single_branch=True, branch=infos.get("branch", "master"), - ) - - -def git_repo_age(path: Path) -> bool | int: - fetch_head = path / ".git" / "FETCH_HEAD" - if fetch_head.exists(): - return int(time.time() - fetch_head.stat().st_mtime) - return False - - -def refresh_cache(app: str, infos: dict[str, str]) -> None: - app_path = app_cache_folder(app) - - # Don't refresh if already refreshed during last hour - age = git_repo_age(app_path) - if age is not False and age < 3600: - return - - try: - repo = Repo(app_path) - - repo.remote("origin").set_url(infos["url"]) - - branch = infos.get("branch", "master") - if repo.active_branch != branch: - all_branches = [str(b) for b in repo.branches] - if branch in all_branches: - repo.git.checkout(branch, "--force") - else: - repo.git.remote("set-branches", "--add", "origin", branch) - repo.remote("origin").fetch(f"{branch}:{branch}") - - repo.remote("origin").fetch(refspec=branch, force=True) - repo.git.reset("--hard", f"origin/{branch}") - except: - # Sometimes there are tmp issue such that the refresh cache .. - # we don't trigger an error unless the cache hasnt been updated since more than 24 hours - age = git_repo_age(app_path) - if age is not False and age < 24 * 3600: - pass - else: - raise +(REPO_APPS_ROOT / "builds").mkdir(exist_ok=True) ################################ # Actual list build management # ################################ +def __build_app_dict(data): + name, info = data + try: + return name, build_app_dict(name, info) + except Exception as err: + logging.error("Error while updating %s: %s", name, err) -def build_catalog(): +def build_base_catalog(): result_dict = {} - for app, infos in progressbar(sorted(catalog.items()), "Processing: ", 40): + with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool: + tasks = pool.imap(__build_app_dict, catalog.items()) - app = app.lower() + for result in tqdm.tqdm(tasks, total=len(catalog.keys()), ascii=" ·#"): + assert result is not None + name, info = result + result_dict[name] = info - try: - app_dict = build_app_dict(app, infos) - except Exception as e: - error("Processing %s failed: %s" % (app, str(e))) - continue + return result_dict - result_dict[app_dict["id"]] = app_dict - ############################# - # Current catalog API v2 # - ############################# +def write_catalog_v2(base_catalog, target_dir: Path) -> None: + result_dict_with_manifest_v1 = copy.deepcopy(base_catalog) + result_dict_with_manifest_v1 = { + name: infos + for name, infos in result_dict_with_manifest_v1.items() + if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2 + } + full_catalog = { + "apps": result_dict_with_manifest_v1, + "categories": categories, + "antifeatures": antifeatures, + } - result_dict_with_manifest_v1 = copy.deepcopy(result_dict) - result_dict_with_manifest_v1 = {name: infos for name, infos in result_dict_with_manifest_v1.items() if float(str(infos["manifest"].get("packaging_format", "")).strip() or "0") < 2} + target_file = target_dir / "apps.json" + target_file.parent.mkdir(parents=True, exist_ok=True) + target_file.open("w", encoding="utf-8").write(json.dumps(full_catalog, sort_keys=True)) - os.system("mkdir -p ./builds/default/v2/") - with open("builds/default/v2/apps.json", "w") as f: - f.write( - json.dumps( - { - "apps": result_dict_with_manifest_v1, - "categories": categories, - "antifeatures": antifeatures, - }, - sort_keys=True, - ) - ) - ############################################# - # Catalog catalog API v3 (with manifest v2) # - ############################################# - - result_dict_with_manifest_v2 = copy.deepcopy(result_dict) +def write_catalog_v3(base_catalog, target_dir: Path) -> None: + result_dict_with_manifest_v2 = copy.deepcopy(base_catalog) for app in result_dict_with_manifest_v2.values(): packaging_format = float(str(app["manifest"].get("packaging_format", "")).strip() or "0") if packaging_format < 2: @@ -218,34 +115,31 @@ def build_catalog(): if "manifest" in app and "resources" in app["manifest"]: del app["manifest"]["resources"] + logos_dir = target_dir / "logos" + logos_dir.mkdir(parents=True, exist_ok=True) for appid, app in result_dict_with_manifest_v2.items(): appid = appid.lower() - if (REPO_APPS_PATH / "logos" / f"{appid}.png").exists(): - logo_hash = subprocess.check_output(["sha256sum", f"logos/{appid}.png"]).strip().decode("utf-8").split()[0] - os.system(f"cp logos/{appid}.png builds/default/v3/logos/{logo_hash}.png") + logo_source = REPO_APPS_ROOT / "logos" / f"{appid}.png" + if logo_source.exists(): + logo_hash = subprocess.check_output(["sha256sum", logo_source]).strip().decode("utf-8").split()[0] + shutil.copyfile(logo_source, logos_dir / f"{logo_hash}.png") # FIXME: implement something to cleanup old logo stuf in the builds/.../logos/ folder somehow else: logo_hash = None app["logo_hash"] = logo_hash - os.system("mkdir -p ./builds/default/v3/") - with open("builds/default/v3/apps.json", "w") as f: - f.write( - json.dumps( - { - "apps": result_dict_with_manifest_v2, - "categories": categories, - "antifeatures": antifeatures, - }, - sort_keys=True, - ) - ) + full_catalog = { + "apps": result_dict_with_manifest_v2, + "categories": categories, + "antifeatures": antifeatures, + } - ############################## - # Version for catalog in doc # - ############################## - os.system("mkdir -p ./builds/default/doc_catalog") + target_file = target_dir / "apps.json" + target_file.parent.mkdir(parents=True, exist_ok=True) + target_file.open("w", encoding="utf-8").write(json.dumps(full_catalog, sort_keys=True)) + +def write_catalog_doc(base_catalog, target_dir: Path) -> None: def infos_for_doc_catalog(infos): level = infos.get("level") if not isinstance(level, int): @@ -267,31 +161,40 @@ def build_catalog(): result_dict_doc = { k: infos_for_doc_catalog(v) - for k, v in result_dict.items() + for k, v in base_catalog.items() if v["state"] == "working" } - with open("builds/default/doc_catalog/apps.json", "w") as f: - f.write( - json.dumps( - {"apps": result_dict_doc, "categories": categories}, sort_keys=True - ) - ) + full_catalog = { + "apps": result_dict_doc, + "categories": categories + } + + target_file = target_dir / "apps.json" + target_file.parent.mkdir(parents=True, exist_ok=True) + target_file.open("w", encoding="utf-8").write(json.dumps(full_catalog, sort_keys=True)) def build_app_dict(app, infos): - # Make sure we have some cache this_app_cache = app_cache_folder(app) - assert this_app_cache.exists(), "No cache yet for %s" % app + assert this_app_cache.exists(), f"No cache yet for {app}" repo = Repo(this_app_cache) - commit_timestamps_for_this_app_in_catalog = \ - repo.git.log("-G", f"cinny", "--first-parent", "--reverse", "--date=unix", - "--format=%cd", "--", "apps.json", "apps.toml") + commits_in_apps_json = Repo(REPO_APPS_ROOT).git.log( + "-S", f"\"{app}\"", "--first-parent", "--reverse", "--date=unix", + "--format=%cd", "--", "apps.json").split("\n") + if len(commits_in_apps_json) > 1: + first_commit = commits_in_apps_json[0] + else: + commits_in_apps_toml = Repo(REPO_APPS_ROOT).git.log( + "-S", f"[{app}]", "--first-parent", "--reverse", "--date=unix", + "--format=%cd", "--", "apps.json", "apps.toml").split("\n") + first_commit = commits_in_apps_toml[0] # Assume the first entry we get (= the oldest) is the time the app was added - infos["added_in_catalog"] = int(commit_timestamps_for_this_app_in_catalog.split("\n")[0]) + infos["added_in_catalog"] = int(first_commit) + # int(commit_timestamps_for_this_app_in_catalog.split("\n")[0]) infos["branch"] = infos.get("branch", "master") infos["revision"] = infos.get("revision", "HEAD") @@ -338,7 +241,7 @@ def build_app_dict(app, infos): "manifest": manifest, "state": infos["state"], "level": infos.get("level", "?"), - "maintained": not 'package-not-maintained' in infos.get('antifeatures', []), + "maintained": 'package-not-maintained' not in infos.get('antifeatures', []), "high_quality": infos.get("high_quality", False), "featured": infos.get("featured", False), "category": infos.get("category", None), @@ -351,5 +254,9 @@ def build_app_dict(app, infos): if __name__ == "__main__": - refresh_all_caches() - build_catalog() + apps_cache_update_all(get_catalog(), parallel=50) + + catalog = build_base_catalog() + write_catalog_v2(catalog, REPO_APPS_ROOT / "builds" / "default" / "v2") + write_catalog_v3(catalog, REPO_APPS_ROOT / "builds" / "default" / "v3") + write_catalog_doc(catalog, REPO_APPS_ROOT / "builds" / "default" / "doc_catalog") From 4118ba6746bad3ba98c8d23beacfea61a7dd4918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 7 Feb 2024 23:45:11 +0100 Subject: [PATCH 02/10] Oops, cleanup legacy file --- tools/appslib/apps_cache.py | 68 ------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 tools/appslib/apps_cache.py diff --git a/tools/appslib/apps_cache.py b/tools/appslib/apps_cache.py deleted file mode 100644 index b8fd1e43..00000000 --- a/tools/appslib/apps_cache.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 - -import logging -from pathlib import Path - -import utils -from git import Repo - - -def apps_cache_path() -> Path: - path = apps_repo_root() / ".apps_cache" - path.mkdir() - return path - - -def app_cache_path(app: str) -> Path: - path = apps_cache_path() / app - path.mkdir() - return path - - -# def refresh_all_caches(catalog: dict[str, dict[str, str]]): -# for app, infos -# pass - - -def app_cache_clone(app: str, infos: dict[str, str]) -> None: - git_depths = { - "notworking": 5, - "inprogress": 20, - "default": 40, - } - - Repo.clone_from( - infos["url"], - to_path=app_cache_path(app), - depth=git_depths.get(infos["state"], git_depths["default"]), - single_branch=True, branch=infos.get("branch", "master"), - ) - - -def app_cache_update(app: str, infos: dict[str, str]) -> None: - app_path = app_cache_path(app) - age = utils.git_repo_age(app_path) - if age is False: - return app_cache_clone(app, infos) - - if age < 3600: - logging.info(f"Skipping {app}, it's been updated recently.") - return - - repo = Repo(app_path) - repo.remote("origin").set_url(infos["url"]) - - branch = infos.get("branch", "master") - if repo.active_branch != branch: - all_branches = [str(b) for b in repo.branches] - if branch in all_branches: - repo.git.checkout(branch, "--force") - else: - repo.git.remote("set-branches", "--add", "origin", branch) - repo.remote("origin").fetch(f"{branch}:{branch}") - - repo.remote("origin").fetch(refspec=branch, force=True) - repo.git.reset("--hard", f"origin/{branch}") - - -def cache_all_apps(catalog: dict[str, dict[str, str]]) -> None: From 950e5907ac9c92dffaa7095aa56e1f76348c5f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:13:11 +0100 Subject: [PATCH 03/10] Commonize code to read catalog, categories etc --- tools/appslib/utils.py | 27 ++++++++++++++++++- tools/catalog_linter.py | 40 +++------------------------- tools/list_builder.py | 58 ++++++++++++++++------------------------- 3 files changed, 53 insertions(+), 72 deletions(-) diff --git a/tools/appslib/utils.py b/tools/appslib/utils.py index f80650e4..46600af6 100644 --- a/tools/appslib/utils.py +++ b/tools/appslib/utils.py @@ -61,7 +61,7 @@ def progressbar( @cache -def get_catalog(working_only=False): +def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]: """Load the app catalog and filter out the non-working ones""" catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) if working_only: @@ -70,3 +70,28 @@ def get_catalog(working_only=False): if infos.get("state") != "notworking" } return catalog + + +@cache +def get_categories() -> dict[str, Any]: + categories_path = REPO_APPS_ROOT / "categories.toml" + return toml.load(categories_path) + + +@cache +def get_antifeatures() -> dict[str, Any]: + antifeatures_path = REPO_APPS_ROOT / "antifeatures.toml" + return toml.load(antifeatures_path) + + +@cache +def get_wishlist() -> dict[str, dict[str, str]]: + wishlist_path = REPO_APPS_ROOT / "wishlist.toml" + return toml.load(wishlist_path) + + +@cache +def get_graveyard() -> dict[str, dict[str, str]]: + wishlist_path = REPO_APPS_ROOT / "graveyard.toml" + return toml.load(wishlist_path) + diff --git a/tools/catalog_linter.py b/tools/catalog_linter.py index 6ce7ef98..53a05aaf 100755 --- a/tools/catalog_linter.py +++ b/tools/catalog_linter.py @@ -3,48 +3,16 @@ import json import sys from difflib import SequenceMatcher -from functools import cache -from pathlib import Path from typing import Any, Dict, Generator, List, Tuple import jsonschema -import toml - -APPS_ROOT = Path(__file__).parent.parent - - -@cache -def get_catalog() -> Dict[str, Dict[str, Any]]: - catalog_path = APPS_ROOT / "apps.toml" - return toml.load(catalog_path) - - -@cache -def get_categories() -> Dict[str, Any]: - categories_path = APPS_ROOT / "categories.toml" - return toml.load(categories_path) - - -@cache -def get_antifeatures() -> Dict[str, Any]: - antifeatures_path = APPS_ROOT / "antifeatures.toml" - return toml.load(antifeatures_path) - - -@cache -def get_wishlist() -> Dict[str, Dict[str, str]]: - wishlist_path = APPS_ROOT / "wishlist.toml" - return toml.load(wishlist_path) - - -@cache -def get_graveyard() -> Dict[str, Dict[str, str]]: - wishlist_path = APPS_ROOT / "graveyard.toml" - return toml.load(wishlist_path) +from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error + get_antifeatures, get_catalog, get_categories, + get_graveyard, get_wishlist) def validate_schema() -> Generator[str, None, None]: - with open(APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: + with open(REPO_APPS_ROOT / "schemas" / "apps.toml.schema.json", encoding="utf-8") as file: apps_catalog_schema = json.load(file) validator = jsonschema.Draft202012Validator(apps_catalog_schema) for error in validator.iter_errors(get_catalog()): diff --git a/tools/list_builder.py b/tools/list_builder.py index 1bb947a0..d069bda6 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -20,41 +20,29 @@ from packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ convert_v1_manifest_to_v2_for_catalog # pylint: disable=import-error from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error - get_catalog, git_repo_age) - -# Automatically enables error-to-xmpp -import appslib.xmpplogger # pylint: disable=import-error - - + get_antifeatures, get_catalog, get_categories) now = time.time() -# Load categories and reformat the structure to have a list with an "id" key -categories = toml.load((REPO_APPS_ROOT / "categories.toml").open("r", encoding="utf-8")) -for category_id, infos in categories.items(): - infos["id"] = category_id - for subtag_id, subtag_infos in infos.get("subtags", {}).items(): - subtag_infos["id"] = subtag_id - infos["subtags"] = list(infos.get('subtags', {}).values()) +@cache +def categories_list(): + # Load categories and reformat the structure to have a list with an "id" key + new_categories = get_categories() + for category_id, infos in new_categories.items(): + infos["id"] = category_id + for subtag_id, subtag_infos in infos.get("subtags", {}).items(): + subtag_infos["id"] = subtag_id + infos["subtags"] = list(infos.get('subtags', {}).values()) + return list(new_categories.values()) -categories = list(categories.values()) -# (Same for antifeatures) -antifeatures = toml.load((REPO_APPS_ROOT / "antifeatures.toml").open("r", encoding="utf-8")) -for antifeature_id, infos in antifeatures.items(): - infos["id"] = antifeature_id -antifeatures = list(antifeatures.values()) - -# Load the app catalog and filter out the non-working ones -catalog = toml.load((REPO_APPS_ROOT / "apps.toml").open("r", encoding="utf-8")) -catalog = { - app: infos for app, infos in catalog.items() if infos.get("state") != "notworking" -} - -my_env = os.environ.copy() -my_env["GIT_TERMINAL_PROMPT"] = "0" - -(REPO_APPS_ROOT / "builds").mkdir(exist_ok=True) +@cache +def antifeatures_list(): + # (Same for antifeatures) + new_antifeatures = get_antifeatures() + for antifeature_id, infos in new_antifeatures.items(): + infos["id"] = antifeature_id + return list(new_antifeatures.values()) ################################ @@ -92,8 +80,8 @@ def write_catalog_v2(base_catalog, target_dir: Path) -> None: } full_catalog = { "apps": result_dict_with_manifest_v1, - "categories": categories, - "antifeatures": antifeatures, + "categories": categories_list(), + "antifeatures": antifeatures_list(), } target_file = target_dir / "apps.json" @@ -130,8 +118,8 @@ def write_catalog_v3(base_catalog, target_dir: Path) -> None: full_catalog = { "apps": result_dict_with_manifest_v2, - "categories": categories, - "antifeatures": antifeatures, + "categories": categories_list(), + "antifeatures": antifeatures_list(), } target_file = target_dir / "apps.json" @@ -166,7 +154,7 @@ def write_catalog_doc(base_catalog, target_dir: Path) -> None: } full_catalog = { "apps": result_dict_doc, - "categories": categories + "categories": categories_list() } target_file = target_dir / "apps.json" From dfd71dc71abc1eebbbe6910beddd67580731c6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:14:35 +0100 Subject: [PATCH 04/10] Cleanup, isort, etc --- tools/appslib/utils.py | 24 -------------------- tools/list_builder.py | 51 +++++++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/tools/appslib/utils.py b/tools/appslib/utils.py index 46600af6..913d16a7 100644 --- a/tools/appslib/utils.py +++ b/tools/appslib/utils.py @@ -36,30 +36,6 @@ def git_repo_age(path: Path) -> bool | int: return False -# Progress bar helper, stolen from https://stackoverflow.com/a/34482761 -def progressbar( - it: list[Any], - prefix: str = "", - size: int = 60, - file: TextIO = sys.stdout) -> Generator[Any, None, None]: - count = len(it) - - def show(j, name=""): - name += " " - x = int(size * j / count) - file.write( - "%s[%s%s] %i/%i %s\r" % (prefix, "#" * x, "." * (size - x), j, count, name) - ) - file.flush() - - show(0) - for i, item in enumerate(it): - yield item - show(i + 1, item[0]) - file.write("\n") - file.flush() - - @cache def get_catalog(working_only: bool = False) -> dict[str, dict[str, Any]]: """Load the app catalog and filter out the non-working ones""" diff --git a/tools/list_builder.py b/tools/list_builder.py index d069bda6..44476f8c 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -2,25 +2,29 @@ import copy import json -import os -import subprocess -import multiprocessing -from pathlib import Path -import time -import shutil -from collections import OrderedDict - -import tqdm import logging +import multiprocessing +import shutil +import subprocess +import time +from collections import OrderedDict +from functools import cache +from pathlib import Path +from typing import Any + import toml +import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm from git import Repo -from app_caches import apps_cache_update_all, app_cache_folder # pylint: disable=import-error +import appslib.xmpplogger # pylint: disable=import-error +from app_caches import app_cache_folder # pylint: disable=import-error +from app_caches import apps_cache_update_all # pylint: disable=import-error +from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error + get_antifeatures, get_catalog, get_categories) from packaging_v2.convert_v1_manifest_to_v2_for_catalog import \ convert_v1_manifest_to_v2_for_catalog # pylint: disable=import-error -from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error - get_antifeatures, get_catalog, get_categories) now = time.time() @@ -49,7 +53,7 @@ def antifeatures_list(): # Actual list build management # ################################ -def __build_app_dict(data): +def __build_app_dict(data) -> tuple[str, dict[str, Any]] | None: name, info = data try: return name, build_app_dict(name, info) @@ -59,14 +63,16 @@ def __build_app_dict(data): def build_base_catalog(): result_dict = {} + catalog = get_catalog(working_only=True) with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool: - tasks = pool.imap(__build_app_dict, catalog.items()) + with logging_redirect_tqdm(): + tasks = pool.imap(__build_app_dict, catalog.items()) - for result in tqdm.tqdm(tasks, total=len(catalog.keys()), ascii=" ·#"): - assert result is not None - name, info = result - result_dict[name] = info + for result in tqdm.tqdm(tasks, total=len(catalog.keys()), ascii=" ·#"): + if result is not None: + name, info = result + result_dict[name] = info return result_dict @@ -96,7 +102,8 @@ def write_catalog_v3(base_catalog, target_dir: Path) -> None: if packaging_format < 2: app["manifest"] = convert_v1_manifest_to_v2_for_catalog(app["manifest"]) - # We also remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) + # We also remove the app install question and resources parts which aint needed anymore + # by webadmin etc (or at least we think ;P) for app in result_dict_with_manifest_v2.values(): if "manifest" in app and "install" in app["manifest"]: del app["manifest"]["install"] @@ -241,10 +248,14 @@ def build_app_dict(app, infos): } -if __name__ == "__main__": +def main() -> None: apps_cache_update_all(get_catalog(), parallel=50) catalog = build_base_catalog() write_catalog_v2(catalog, REPO_APPS_ROOT / "builds" / "default" / "v2") write_catalog_v3(catalog, REPO_APPS_ROOT / "builds" / "default" / "v3") write_catalog_doc(catalog, REPO_APPS_ROOT / "builds" / "default" / "doc_catalog") + + +if __name__ == "__main__": + main() From dd50e4dedb643d89eec04cb6b2920388b307929d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:15:08 +0100 Subject: [PATCH 05/10] don't do automatic activation of xmpplogger --- tools/appslib/xmpplogger.py | 4 +++- tools/list_builder.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/appslib/xmpplogger.py b/tools/appslib/xmpplogger.py index 69c48148..c13faef6 100644 --- a/tools/appslib/xmpplogger.py +++ b/tools/appslib/xmpplogger.py @@ -30,4 +30,6 @@ class XmppLogHandler(logging.Handler): logging.getLogger().handlers.append(handler) -XmppLogHandler.add(logging.ERROR) +def enable(): + """Enables the XmppLogHandler""" + XmppLogHandler.add(logging.ERROR) diff --git a/tools/list_builder.py b/tools/list_builder.py index 44476f8c..586d49bc 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -249,6 +249,7 @@ def build_app_dict(app, infos): def main() -> None: + appslib.xmpplogger.enable() apps_cache_update_all(get_catalog(), parallel=50) catalog = build_base_catalog() From f893187abdd1c94c54a891edd676c98587df3650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:18:59 +0100 Subject: [PATCH 06/10] Rename xmpplogger into logging_sender --- tools/appslib/{xmpplogger.py => logging_sender.py} | 7 ++++--- tools/list_builder.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) rename tools/appslib/{xmpplogger.py => logging_sender.py} (80%) diff --git a/tools/appslib/xmpplogger.py b/tools/appslib/logging_sender.py similarity index 80% rename from tools/appslib/xmpplogger.py rename to tools/appslib/logging_sender.py index c13faef6..9cd9c801 100644 --- a/tools/appslib/xmpplogger.py +++ b/tools/appslib/logging_sender.py @@ -6,13 +6,14 @@ import logging import logging.handlers -class XmppLogHandler(logging.Handler): +class LogSenderHandler(logging.Handler): def __init__(self): logging.Handler.__init__(self) self.is_logging = False def emit(self, record): if which("sendxmpppy") is None: + logging.warning("Could not send error via xmpp.") return msg = f"[Applist builder error] {record.msg}" @@ -31,5 +32,5 @@ class XmppLogHandler(logging.Handler): def enable(): - """Enables the XmppLogHandler""" - XmppLogHandler.add(logging.ERROR) + """Enables the LogSenderHandler""" + LogSenderHandler.add(logging.ERROR) diff --git a/tools/list_builder.py b/tools/list_builder.py index 586d49bc..cf746d95 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -17,7 +17,7 @@ import tqdm from tqdm.contrib.logging import logging_redirect_tqdm from git import Repo -import appslib.xmpplogger # pylint: disable=import-error +import appslib.logging_sender # pylint: disable=import-error from app_caches import app_cache_folder # pylint: disable=import-error from app_caches import apps_cache_update_all # pylint: disable=import-error from appslib.utils import (REPO_APPS_ROOT, # pylint: disable=import-error @@ -249,7 +249,7 @@ def build_app_dict(app, infos): def main() -> None: - appslib.xmpplogger.enable() + appslib.logging_sender.enable() apps_cache_update_all(get_catalog(), parallel=50) catalog = build_base_catalog() From fc327045657ff00c5b38ae553f807d83a35f0eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:20:07 +0100 Subject: [PATCH 07/10] woops, end of file --- tools/appslib/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/appslib/utils.py b/tools/appslib/utils.py index 913d16a7..d387b1d4 100644 --- a/tools/appslib/utils.py +++ b/tools/appslib/utils.py @@ -70,4 +70,3 @@ def get_wishlist() -> dict[str, dict[str, str]]: def get_graveyard() -> dict[str, dict[str, str]]: wishlist_path = REPO_APPS_ROOT / "graveyard.toml" return toml.load(wishlist_path) - From b905981f710767e442ff7b7c611a26f2abce30db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:26:30 +0100 Subject: [PATCH 08/10] Add required python packages to github workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df76b84e..f7466b90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: python-version: 3.11 - name: Install toml python lib run: | - pip3 install toml jsonschema + pip3 install toml jsonschema gitpython tqdm - name: Check TOML validity for apps.toml run: | python3 -c "import toml; toml.load(open('apps.toml'))" From 7f3f628b5de30a3fb0a32c810cb10ad1fe158a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:30:12 +0100 Subject: [PATCH 09/10] Add test that builds the catalog --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f7466b90..ab0aede4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,4 +23,7 @@ jobs: python3 -c "import toml; toml.load(open('apps.toml'))" - name: Check all working apps have consistent app id / app url and categories run: | - python3 tools/catalog_linter.py + ./tools/catalog_linter.py + - name: Check the generation of the app catalog + run: | + ./tools/list_builder.py From 5e692f37d7498587895d82a8b8bbc562bbed259d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 8 Feb 2024 22:34:48 +0100 Subject: [PATCH 10/10] Fix scripts shebangs --- rebuild.sh | 2 +- tools/autopatches/autopatch.py | 2 +- tools/autopatches/patches/add-cpe/patch.sh | 2 +- tools/list_builder.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebuild.sh b/rebuild.sh index 7c4780fe..3dc9b232 100644 --- a/rebuild.sh +++ b/rebuild.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash workdir=$(dirname "$0") log=$workdir/app_list_auto_update.log diff --git a/tools/autopatches/autopatch.py b/tools/autopatches/autopatch.py index 6f52d309..34ce499a 100755 --- a/tools/autopatches/autopatch.py +++ b/tools/autopatches/autopatch.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import json import os diff --git a/tools/autopatches/patches/add-cpe/patch.sh b/tools/autopatches/patches/add-cpe/patch.sh index 2b6b2d6c..ad27a117 100644 --- a/tools/autopatches/patches/add-cpe/patch.sh +++ b/tools/autopatches/patches/add-cpe/patch.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import csv import json diff --git a/tools/list_builder.py b/tools/list_builder.py index cf746d95..e48ca57d 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import copy import json