From 751ff67fd94d506509f9d42a26e6f8a25a719de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 13:08:31 +0100 Subject: [PATCH 01/12] Add python linting in CI with Ruff. Fix the on: push/pull_request for other actions. --- .github/workflows/autoblack.yml | 5 +++-- .github/workflows/python_lint.yml | 25 +++++++++++++++++++++++++ .github/workflows/shellcheck.yml | 1 + custom-catalog/catalog_manager.py | 12 +++++++----- 4 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/python_lint.yml diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml index 761222b..8a1da50 100644 --- a/.github/workflows/autoblack.yml +++ b/.github/workflows/autoblack.yml @@ -1,8 +1,9 @@ name: Check / auto apply Black on: push: - branches: - - master + branches: + - master + jobs: black: name: Check / auto apply black diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 0000000..b84305a --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,25 @@ +name: Python Ruff Lint and commit + +on: + push: + branches: [master] + pull_request: + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + - name: Install Ruff + run: pip install ruff + + - name: Ruff check and fix + run: | + ruff check . + ruff fix . + + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: 'Python style fixes from Ruff' diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index d4e239c..e6e5d05 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -2,6 +2,7 @@ name: Run Shellcheck on push and PR on: push: + branches: [master] pull_request: jobs: diff --git a/custom-catalog/catalog_manager.py b/custom-catalog/catalog_manager.py index 09baab7..b8a14b5 100755 --- a/custom-catalog/catalog_manager.py +++ b/custom-catalog/catalog_manager.py @@ -1,13 +1,14 @@ #!/usr/bin/python3 -import sys -import os import json -import toml -import yaml +import os +import sys import time from collections import OrderedDict +import toml +import yaml + CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" assert os.path.exists( CATALOG_LIST_PATH @@ -44,7 +45,8 @@ def build(folder=DEFAULT_APPS_FOLDER): apps[app_dict["id"]] = app_dict - # We also remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) + # We also remove the app install question and resources parts which aint needed + # anymore by webadmin etc (or at least we think ;P) for app in apps.values(): if "manifest" in app and "install" in app["manifest"]: del app["manifest"]["install"] From e03e960249d06de3e7a07d9aba0bf0e9d1b1fa5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:05:09 +0100 Subject: [PATCH 02/12] Reorganize github workflows: one check on push+pr, one check+format then commit, only on push on master --- .github/workflows/autoblack.yml | 36 ------------------- .../{python_lint.yml => python-check.yml} | 16 ++++----- .github/workflows/python-format.yml | 32 +++++++++++++++++ pyproject.toml | 21 +++++++++++ 4 files changed, 61 insertions(+), 44 deletions(-) delete mode 100644 .github/workflows/autoblack.yml rename .github/workflows/{python_lint.yml => python-check.yml} (51%) create mode 100644 .github/workflows/python-format.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml deleted file mode 100644 index 8a1da50..0000000 --- a/.github/workflows/autoblack.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Check / auto apply Black -on: - push: - branches: - - master - -jobs: - black: - name: Check / auto apply black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check files using the black formatter - uses: psf/black@stable - id: black - with: - options: "." - continue-on-error: true - - shell: pwsh - id: check_files_changed - run: | - # Diff HEAD with the previous commit - $diff = git diff - $HasDiff = $diff.Length -gt 0 - Write-Host "::set-output name=files_changed::$HasDiff" - - name: Create Pull Request - if: steps.check_files_changed.outputs.files_changed == 'true' - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - title: "Format Python code with Black" - commit-message: ":art: Format Python code with Black" - body: | - This pull request uses the [psf/black](https://github.com/psf/black) formatter. - base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch - branch: actions/black diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python-check.yml similarity index 51% rename from .github/workflows/python_lint.yml rename to .github/workflows/python-check.yml index b84305a..9b7dabb 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python-check.yml @@ -1,9 +1,9 @@ -name: Python Ruff Lint and commit +name: Python Lint with Ruff and Mypy on: + pull_request: push: branches: [master] - pull_request: jobs: ruff: @@ -12,14 +12,14 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - name: Install Ruff + - name: Install Ruff and Mypy run: pip install ruff - - name: Ruff check and fix + - name: Ruff check run: | ruff check . - ruff fix . - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: 'Python style fixes from Ruff' + - name: mypy check + run: | + yes | mypy src --install-types || true + mypy . diff --git a/.github/workflows/python-format.yml b/.github/workflows/python-format.yml new file mode 100644 index 0000000..c1199e1 --- /dev/null +++ b/.github/workflows/python-format.yml @@ -0,0 +1,32 @@ +name: Python formatting with Black and Ruff + +on: + push: + branches: [master] + +jobs: + format: + name: Formatting with Black and Ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check files using the black formatter + uses: psf/black@stable + id: black + with: + options: "." + continue-on-error: true + + - uses: actions/setup-python@v2 + - name: Install Ruff + run: pip install ruff + + - name: Ruff check and fix + run: | + ruff check . + ruff check --fix . + + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: 'Python formatting fixes from Black and Ruff' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a015222 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[tool.black] +line-length = 120 + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +select = [ + "F", # pyflakes + "E", # pycodestyle + "W", # pycodestyle + "I", # isort + "N", # pep8-naming + "B", # flake8-ubgbear + "ANN", # flake8-annotations + "Q", # flake8-quotes + "PTH", # flake8-use-pathlib + "UP", # pyupgrade, +] + +[tool.mypy] From 0829da0f4635208b13567cd4930539f74ba14831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:27:23 +0100 Subject: [PATCH 03/12] Fix ruff errors --- custom-catalog/catalog_manager.py | 58 ++++++++++++++----------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/custom-catalog/catalog_manager.py b/custom-catalog/catalog_manager.py index b8a14b5..70874ba 100755 --- a/custom-catalog/catalog_manager.py +++ b/custom-catalog/catalog_manager.py @@ -5,14 +5,14 @@ import os import sys import time from collections import OrderedDict +from pathlib import Path +from typing import Any import toml import yaml -CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" -assert os.path.exists( - CATALOG_LIST_PATH -), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" +CATALOG_LIST_PATH = Path("/etc/yunohost/apps_catalog.yml").resolve() +assert CATALOG_LIST_PATH.exists(), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" now = time.time() my_env = os.environ.copy() @@ -22,13 +22,14 @@ DEFAULT_APPS_FOLDER = "/ynh-dev/custom-catalog/" DEFAULT_APP_BRANCH = "master" -def build(folder=DEFAULT_APPS_FOLDER): - assert os.path.exists(folder), f"'{folder}' doesn't exists." +def build(folder: Path | str = DEFAULT_APPS_FOLDER) -> None: + folder = Path(folder) + assert folder.exists(), f"'{folder}' doesn't exists." - app_list_path = os.path.join(folder, "apps.json") - assert os.path.exists(app_list_path), "no 'apps.json' app list found." + app_list_path = folder / "apps.json" + assert app_list_path.exists(), "no 'apps.json' app list found." - with open(app_list_path) as f: + with app_list_path.open() as f: app_list = json.load(f) apps = {} @@ -53,30 +54,30 @@ def build(folder=DEFAULT_APPS_FOLDER): if "manifest" in app and "resources" in app["manifest"]: del app["manifest"]["resources"] - output_file = os.path.join(folder, "catalog.json") + output_file = folder / "catalog.json" data = { "apps": apps, "from_api_version": 3, } - with open(output_file, "w") as f: + with output_file.open("w") as f: f.write(json.dumps(data, sort_keys=True, indent=2)) if fail: sys.exit(1) -def build_app_dict(app, infos, folder): - app_folder = os.path.join(folder, app + "_ynh") +def build_app_dict(app: str, infos: dict[str, Any], folder: Path) -> dict[str, Any]: + app_folder = folder / f"{app}_ynh" # Build the dict with all the infos - manifest_toml = os.path.join(app_folder, "manifest.toml") - manifest_json = os.path.join(app_folder, "manifest.json") - if os.path.exists(manifest_toml): - with open(manifest_toml) as f: + manifest_toml = app_folder / "manifest.toml" + manifest_json = app_folder / "manifest.json" + if manifest_toml.exists(): + with manifest_toml.open() as f: manifest = toml.load(f, _dict=OrderedDict) else: - with open(manifest_json) as f: + with manifest_json.open() as f: manifest = json.load(f, _dict=OrderedDict) return { @@ -96,32 +97,27 @@ def build_app_dict(app, infos, folder): "category": infos.get("category", None), "subtags": infos.get("subtags", []), "potential_alternative_to": infos.get("potential_alternative_to", []), - "antifeatures": list( - set( - list(manifest.get("antifeatures", {}).keys()) - + infos.get("antifeatures", []) - ) - ), + "antifeatures": list(set(list(manifest.get("antifeatures", {}).keys()) + infos.get("antifeatures", []))), } -def reset(): - with open(CATALOG_LIST_PATH, "w") as f: +def reset() -> None: + with CATALOG_LIST_PATH.open("w") as f: catalog_list = [{"id": "default", "url": "https://app.yunohost.org/default/"}] yaml.safe_dump(catalog_list, f, default_flow_style=False) -def add(): - with open(CATALOG_LIST_PATH) as f: +def add() -> None: + with CATALOG_LIST_PATH.open("r") as f: catalog_list = yaml.load(f, Loader=yaml.FullLoader) ids = [catalog["id"] for catalog in catalog_list] if "custom" not in ids: catalog_list.append({"id": "custom", "url": None}) - with open(CATALOG_LIST_PATH, "w") as f: + with CATALOG_LIST_PATH.open("w") as f: yaml.safe_dump(catalog_list, f, default_flow_style=False) -def override(): - with open(CATALOG_LIST_PATH, "w") as f: +def override() -> None: + with CATALOG_LIST_PATH.open("w") as f: catalog_list = [{"id": "custom", "url": None}] yaml.safe_dump(catalog_list, f, default_flow_style=False) From 6e03cebf483572c370d9622c877d1d30f97e105d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:38:36 +0100 Subject: [PATCH 04/12] Oops, install mypy on CI --- .github/workflows/python-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-check.yml b/.github/workflows/python-check.yml index 9b7dabb..893a7ba 100644 --- a/.github/workflows/python-check.yml +++ b/.github/workflows/python-check.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/setup-python@v2 - name: Install Ruff and Mypy - run: pip install ruff + run: pip install ruff mypy - name: Ruff check run: | From 8886c1c99486cc8654bfc43ab4a515235dfe78c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:39:28 +0100 Subject: [PATCH 05/12] Fix mypy call on CI --- .github/workflows/python-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-check.yml b/.github/workflows/python-check.yml index 893a7ba..87d404f 100644 --- a/.github/workflows/python-check.yml +++ b/.github/workflows/python-check.yml @@ -21,5 +21,5 @@ jobs: - name: mypy check run: | - yes | mypy src --install-types || true + yes | mypy . --install-types || true mypy . From 1cb128d1fc943a0a37558f0332a148b11b888c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:41:42 +0100 Subject: [PATCH 06/12] Split mypy steps in CI --- .github/workflows/python-check.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-check.yml b/.github/workflows/python-check.yml index 87d404f..834f64e 100644 --- a/.github/workflows/python-check.yml +++ b/.github/workflows/python-check.yml @@ -19,7 +19,8 @@ jobs: run: | ruff check . - - name: mypy check - run: | - yes | mypy . --install-types || true - mypy . + - name: Mypy install types + run: yes | mypy . --install-types || true + + - name: Mypy check + run: mypy . From 748fb56d2cc97de5f4c312a18a1fbf793de31b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Thu, 14 Mar 2024 14:44:31 +0100 Subject: [PATCH 07/12] Update gitignore for python cache directories --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 4ca7c29..c7e49c6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ ssowat # Folders apps backup + + +__pycache__/ +.mypy_cache/ +.ruff_cache/ From 6abf422a0808e30f147311010866d5b1ce18379e Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 14 Mar 2024 16:25:58 +0100 Subject: [PATCH 08/12] Update actions/setup-python --- .github/workflows/python-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-format.yml b/.github/workflows/python-format.yml index c1199e1..f030616 100644 --- a/.github/workflows/python-format.yml +++ b/.github/workflows/python-format.yml @@ -18,7 +18,7 @@ jobs: options: "." continue-on-error: true - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 - name: Install Ruff run: pip install ruff From e22af3852bb0b8ed97b45a7568c9ecce1df934f6 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 14 Mar 2024 16:29:00 +0100 Subject: [PATCH 09/12] Update python-check.yml Update actions/checkout and actions/setup-python --- .github/workflows/python-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-check.yml b/.github/workflows/python-check.yml index 834f64e..d3af666 100644 --- a/.github/workflows/python-check.yml +++ b/.github/workflows/python-check.yml @@ -9,9 +9,9 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 - name: Install Ruff and Mypy run: pip install ruff mypy From 2be866d2b55324390b166dce042bc3323a41f389 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 14 Mar 2024 16:30:25 +0100 Subject: [PATCH 10/12] Update stefanzweifel/git-auto-commit-action --- .github/workflows/python-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-format.yml b/.github/workflows/python-format.yml index f030616..2f03437 100644 --- a/.github/workflows/python-format.yml +++ b/.github/workflows/python-format.yml @@ -27,6 +27,6 @@ jobs: ruff check . ruff check --fix . - - uses: stefanzweifel/git-auto-commit-action@v4 + - uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: 'Python formatting fixes from Black and Ruff' From b92b7950e3e8b5e6b97a26cd0e9e0f45de8672be Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 14 Mar 2024 17:01:00 +0100 Subject: [PATCH 11/12] Update actions/checkout --- .github/workflows/shellcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index e6e5d05..cd66aa5 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -10,7 +10,7 @@ jobs: name: Shellcheck runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master with: From 49dbf3846152389ae9d60a9a2d772e967ab1d177 Mon Sep 17 00:00:00 2001 From: OniriCorpe Date: Thu, 14 Mar 2024 17:05:09 +0100 Subject: [PATCH 12/12] specify python-version --- .github/workflows/python-check.yml | 2 ++ .github/workflows/python-format.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/python-check.yml b/.github/workflows/python-check.yml index d3af666..3c3445b 100644 --- a/.github/workflows/python-check.yml +++ b/.github/workflows/python-check.yml @@ -12,6 +12,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + with: + python-version: "3.9" - name: Install Ruff and Mypy run: pip install ruff mypy diff --git a/.github/workflows/python-format.yml b/.github/workflows/python-format.yml index 2f03437..8ff8d9e 100644 --- a/.github/workflows/python-format.yml +++ b/.github/workflows/python-format.yml @@ -19,6 +19,8 @@ jobs: continue-on-error: true - uses: actions/setup-python@v5 + with: + python-version: "3.9" - name: Install Ruff run: pip install ruff