diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 23a0075de..eb34de38e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,142 +1,22 @@ stages: - - postinstall + - build + - install - tests - lint + - doc -######################################## -# POSTINSTALL -######################################## +default: + tags: + - yunohost-ci + # All jobs are interruptible by default + interruptible: true -postinstall: - image: before-postinstall - stage: postinstall - script: - - apt install --no-install-recommends -y $(cat debian/control | grep "^Depends" -A50 | grep "Recommends:" -B50 | grep "^ *," | grep -o -P "[\w\-]{3,}") - - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns +variables: + YNH_BUILD_DIR: "ynh-build" -######################################## -# TESTS -######################################## - -.test-stage: - image: after-postinstall - stage: tests - before_script: - - apt-get install python-pip -y - - mkdir -p .pip - - pip install -U pip - - hash -d pip - - pip --cache-dir=.pip install pytest pytest-sugar pytest-mock requests-mock mock - - export PYTEST_ADDOPTS="--color=yes" - cache: - paths: - - .pip - - src/yunohost/tests/apps - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - -root-tests: - extends: .test-stage - script: - - py.test tests - -test-apps: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_apps.py - -test-appscatalog: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_appscatalog.py - -test-appurl: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_appurl.py - -test-apps-arguments-parsing: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_apps_arguments_parsing.py - -test-backuprestore: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_backuprestore.py - -test-changeurl: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_changeurl.py - -test-permission: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_permission.py - -test-settings: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_settings.py - -test-user-group: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_user-group.py - -test-regenconf: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_regenconf.py - -test-service: - extends: .test-stage - script: - - cd src/yunohost - - py.test tests/test_service.py - -######################################## -# LINTER -######################################## - -.lint-stage: - image: before-postinstall - stage: lint - before_script: - - apt-get install python-pip -y - - mkdir -p .pip - - pip install -U pip - - hash -d pip - - pip --cache-dir=.pip install tox - cache: - paths: - - .pip - - .tox - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - -lint: - extends: .lint-stage - allow_failure: true - script: - - tox -e lint - -invalidcode: - extends: .lint-stage - script: - - tox -e invalidcode - -# Disabled, waiting for buster -#format-check: -# extends: .lint-stage -# script: -# - black --check --diff +include: + - local: .gitlab/ci/build.gitlab-ci.yml + - local: .gitlab/ci/install.gitlab-ci.yml + - local: .gitlab/ci/test.gitlab-ci.yml + - local: .gitlab/ci/lint.gitlab-ci.yml + - local: .gitlab/ci/doc.gitlab-ci.yml diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml new file mode 100644 index 000000000..67232ba1f --- /dev/null +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -0,0 +1,52 @@ +.build-stage: + stage: build + image: "before-install" + variables: + YNH_SOURCE: "https://github.com/yunohost" + before_script: + - mkdir -p $YNH_BUILD_DIR + artifacts: + paths: + - $YNH_BUILD_DIR/*.deb + +.build_script: &build_script + - cd $YNH_BUILD_DIR/$PACKAGE + - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null) + - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)" + - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build." + - debuild --no-lintian -us -uc + +######################################## +# BUILD DEB +######################################## + +build-yunohost: + extends: .build-stage + variables: + PACKAGE: "yunohost" + script: + - git ls-files | xargs tar -czf archive.tar.gz + - mkdir -p $YNH_BUILD_DIR/$PACKAGE + - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE + - rm archive.tar.gz + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE + - *build_script + + +build-ssowat: + extends: .build-stage + variables: + PACKAGE: "ssowat" + script: + - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE + - *build_script + +build-moulinette: + extends: .build-stage + variables: + PACKAGE: "moulinette" + script: + - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE + - *build_script diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml new file mode 100644 index 000000000..b246cc238 --- /dev/null +++ b/.gitlab/ci/doc.gitlab-ci.yml @@ -0,0 +1,14 @@ +######################################## +# DOC +######################################## + +generate-helpers-doc: + stage: doc + image: "before-install" + needs: [] + script: + - cd doc + - python generate_helper_doc.py + artifacts: + paths: + - doc/helpers.html diff --git a/.gitlab/ci/install.gitlab-ci.yml b/.gitlab/ci/install.gitlab-ci.yml new file mode 100644 index 000000000..8363b6e58 --- /dev/null +++ b/.gitlab/ci/install.gitlab-ci.yml @@ -0,0 +1,27 @@ +.install-stage: + stage: install + needs: + - job: build-yunohost + artifacts: true + - job: build-ssowat + artifacts: true + - job: build-moulinette + artifacts: true + +######################################## +# INSTALL DEB +######################################## + +upgrade: + extends: .install-stage + image: "after-install" + script: + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb + + +install-postinstall: + extends: .install-stage + image: "before-install" + script: + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb + - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml new file mode 100644 index 000000000..c6967d5a5 --- /dev/null +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -0,0 +1,24 @@ +######################################## +# LINTER +######################################## + +lint: + stage: lint + image: "before-install" + needs: [] + allow_failure: true + script: + - tox -e lint + +invalidcode: + stage: lint + image: "before-install" + needs: [] + script: + - tox -e invalidcode + +# Disabled, waiting for buster +#format-check: +# extends: .lint-stage +# script: +# - black --check --diff diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml new file mode 100644 index 000000000..b52e08085 --- /dev/null +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -0,0 +1,119 @@ +.install_debs: &install_debs + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb + +.test-stage: + stage: tests + image: "after-install" + variables: + PYTEST_ADDOPTS: "--color=yes" + before_script: + - *install_debs + cache: + paths: + - src/yunohost/tests/apps + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + needs: + - job: build-yunohost + artifacts: true + - job: build-ssowat + artifacts: true + - job: build-moulinette + artifacts: true + - job: upgrade + + +######################################## +# TESTS +######################################## + +full-tests: + stage: tests + image: "before-install" + variables: + PYTEST_ADDOPTS: "--color=yes" + before_script: + - *install_debs + - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns + script: + - pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml + needs: + - job: build-yunohost + artifacts: true + - job: build-ssowat + artifacts: true + - job: build-moulinette + artifacts: true + artifacts: + reports: + junit: report.xml + +root-tests: + extends: .test-stage + script: + - py.test tests + +test-apps: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_apps.py + +test-appscatalog: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_appscatalog.py + +test-appurl: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_appurl.py + +test-apps-arguments-parsing: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_apps_arguments_parsing.py + +test-backuprestore: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_backuprestore.py + +test-changeurl: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_changeurl.py + +test-permission: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_permission.py + +test-settings: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_settings.py + +test-user-group: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_user-group.py + +test-regenconf: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_regenconf.py + +test-service: + extends: .test-stage + script: + - cd src/yunohost + - py.test tests/test_service.py diff --git a/pytest.ini b/pytest.ini index fb4d5b265..709e0e0b9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,3 +10,5 @@ markers = with_legacy_app_installed with_backup_recommended_app_installed_with_ynh_restore with_permission_app_installed +filterwarnings = + ignore::urllib3.exceptions.InsecureRequestWarning \ No newline at end of file diff --git a/src/yunohost/tests/conftest.py b/src/yunohost/tests/conftest.py index 073c880f8..d5f3ed186 100644 --- a/src/yunohost/tests/conftest.py +++ b/src/yunohost/tests/conftest.py @@ -19,6 +19,9 @@ def clone_test_app(request): else: os.system("cd %s/apps && git pull > /dev/null 2>&1" % cwd) +def get_test_apps_dir(): + cwd = os.path.split(os.path.realpath(__file__))[0] + return os.path.join(cwd, "apps") @contextmanager def message(mocker, key, **kwargs): diff --git a/src/yunohost/tests/test_apps.py b/src/yunohost/tests/test_apps.py index c2c7b8415..0f4c3749a 100644 --- a/src/yunohost/tests/test_apps.py +++ b/src/yunohost/tests/test_apps.py @@ -4,7 +4,7 @@ import pytest import shutil import requests -from conftest import message, raiseYunohostError +from conftest import message, raiseYunohostError, get_test_apps_dir from moulinette.utils.filesystem import mkdir @@ -150,7 +150,7 @@ def app_is_exposed_on_http(domain, path, message_in_page): def install_legacy_app(domain, path, public=True): app_install( - "./tests/apps/legacy_app_ynh", + os.path.join(get_test_apps_dir(), "legacy_app_ynh"), args="domain=%s&path=%s&is_public=%s" % (domain, path, 1 if public else 0), force=True, ) @@ -159,14 +159,14 @@ def install_legacy_app(domain, path, public=True): def install_full_domain_app(domain): app_install( - "./tests/apps/full_domain_app_ynh", args="domain=%s" % domain, force=True + os.path.join(get_test_apps_dir(), "full_domain_app_ynh"), args="domain=%s" % domain, force=True ) def install_break_yo_system(domain, breakwhat): app_install( - "./tests/apps/break_yo_system_ynh", + os.path.join(get_test_apps_dir(), "break_yo_system_ynh"), args="domain=%s&breakwhat=%s" % (domain, breakwhat), force=True, ) @@ -376,7 +376,7 @@ def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain): with pytest.raises(YunohostError): with message(mocker, "app_action_broke_system"): - app_upgrade("break_yo_system", file="./tests/apps/break_yo_system_ynh") + app_upgrade("break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh")) def test_failed_multiple_app_upgrade(mocker, secondary_domain): @@ -389,7 +389,7 @@ def test_failed_multiple_app_upgrade(mocker, secondary_domain): app_upgrade( ["break_yo_system", "legacy_app"], file={ - "break_yo_system": "./tests/apps/break_yo_system_ynh", - "legacy": "./tests/apps/legacy_app_ynh", + "break_yo_system": os.path.join(get_test_apps_dir(), "break_yo_system_ynh"), + "legacy": os.path.join(get_test_apps_dir(), "legacy_app_ynh"), }, ) diff --git a/src/yunohost/tests/test_appurl.py b/src/yunohost/tests/test_appurl.py index b0a47d89f..8c30a8ec7 100644 --- a/src/yunohost/tests/test_appurl.py +++ b/src/yunohost/tests/test_appurl.py @@ -1,4 +1,7 @@ import pytest +import os + +from conftest import get_test_apps_dir from yunohost.utils.error import YunohostError from yunohost.app import app_install, app_remove @@ -43,19 +46,19 @@ def test_urlavailable(): def test_registerurl(): - app_install("./tests/apps/register_url_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"), args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"), force=True) assert not domain_url_available(maindomain, "/urlregisterapp") # Try installing at same location with pytest.raises(YunohostError): - app_install("./tests/apps/register_url_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"), args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"), force=True) def test_registerurl_baddomain(): with pytest.raises(YunohostError): - app_install("./tests/apps/register_url_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"), args="domain=%s&path=%s" % ("yolo.swag", "/urlregisterapp"), force=True) diff --git a/src/yunohost/tests/test_backuprestore.py b/src/yunohost/tests/test_backuprestore.py index 026e87c95..5cffc186e 100644 --- a/src/yunohost/tests/test_backuprestore.py +++ b/src/yunohost/tests/test_backuprestore.py @@ -3,7 +3,7 @@ import os import shutil import subprocess -from conftest import message, raiseYunohostError +from conftest import message, raiseYunohostError, get_test_apps_dir from yunohost.app import app_install, app_remove, app_ssowatconf from yunohost.app import _is_installed @@ -117,9 +117,9 @@ def app_is_installed(app): def backup_test_dependencies_are_met(): # Dummy test apps (or backup archives) - assert os.path.exists("./tests/apps/backup_wordpress_from_2p4") - assert os.path.exists("./tests/apps/legacy_app_ynh") - assert os.path.exists("./tests/apps/backup_recommended_app_ynh") + assert os.path.exists(os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4")) + assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) + assert os.path.exists(os.path.join(get_test_apps_dir(), "backup_recommended_app_ynh")) return True @@ -175,7 +175,7 @@ def uninstall_test_apps_if_needed(): def install_app(app, path, additionnal_args=""): - app_install("./tests/apps/%s" % app, + app_install(os.path.join(get_test_apps_dir(), app), args="domain=%s&path=%s%s" % (maindomain, path, additionnal_args), force=True) @@ -184,22 +184,22 @@ def add_archive_wordpress_from_2p4(): os.system("mkdir -p /home/yunohost.backup/archives") - os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.info.json \ - /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json") + os.system("cp " + os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4/backup.info.json") + \ + " /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json") - os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.tar.gz \ - /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz") + os.system("cp " + os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4/backup.tar.gz") + \ + " /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz") def add_archive_system_from_2p4(): os.system("mkdir -p /home/yunohost.backup/archives") - os.system("cp ./tests/apps/backup_system_from_2p4/backup.info.json \ - /home/yunohost.backup/archives/backup_system_from_2p4.info.json") + os.system("cp " + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.info.json") + \ + " /home/yunohost.backup/archives/backup_system_from_2p4.info.json") - os.system("cp ./tests/apps/backup_system_from_2p4/backup.tar.gz \ - /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz") + os.system("cp " + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.tar.gz") + \ + " /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz") # # System backup # diff --git a/src/yunohost/tests/test_changeurl.py b/src/yunohost/tests/test_changeurl.py index 8888dd9e9..b1bedc7eb 100644 --- a/src/yunohost/tests/test_changeurl.py +++ b/src/yunohost/tests/test_changeurl.py @@ -1,6 +1,9 @@ import pytest import time import requests +import os + +from conftest import get_test_apps_dir from yunohost.app import app_install, app_change_url, app_remove, app_map from yunohost.domain import _get_maindomain @@ -20,7 +23,7 @@ def teardown_function(function): def install_changeurl_app(path): - app_install("./tests/apps/change_url_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "change_url_app_ynh"), args="domain=%s&path=%s" % (maindomain, path), force=True) diff --git a/src/yunohost/tests/test_permission.py b/src/yunohost/tests/test_permission.py index b6b7d9577..82ec151f3 100644 --- a/src/yunohost/tests/test_permission.py +++ b/src/yunohost/tests/test_permission.py @@ -1,7 +1,8 @@ import requests import pytest +import os -from conftest import message, raiseYunohostError +from conftest import message, raiseYunohostError, get_test_apps_dir from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map, _installed_apps from yunohost.user import user_list, user_create, user_delete, \ @@ -418,7 +419,7 @@ def test_permission_remove_url(): def test_permission_app_install(): - app_install("./tests/apps/permissions_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&path=%s&is_public=0&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True) res = user_permission_list(full=True)['permissions'] @@ -446,7 +447,7 @@ def test_permission_app_install(): def test_permission_app_remove(): - app_install("./tests/apps/permissions_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&path=%s&is_public=0&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True) app_remove("permissions_app") @@ -456,7 +457,7 @@ def test_permission_app_remove(): def test_permission_app_change_url(): - app_install("./tests/apps/permissions_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&path=%s&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True) # FIXME : should rework this test to look for differences in the generated app map / app tiles ... @@ -475,7 +476,7 @@ def test_permission_app_change_url(): def test_permission_app_propagation_on_ssowat(): - app_install("./tests/apps/permissions_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&path=%s&is_public=1&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True) res = user_permission_list(full=True)['permissions'] @@ -505,7 +506,7 @@ def test_permission_app_propagation_on_ssowat(): def test_permission_legacy_app_propagation_on_ssowat(): - app_install("./tests/apps/legacy_app_ynh", + app_install(os.path.join(get_test_apps_dir(), "legacy_app_ynh"), args="domain=%s&path=%s" % (maindomain, "/legacy"), force=True) # App is configured as public by default using the legacy unprotected_uri mechanics diff --git a/tox.ini b/tox.ini index 8d033367b..4598ad3d3 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps = pytest >= 4.6.3, < 5.0 pyyaml >= 5.1.2, < 6.0 flake8 >= 3.7.9, < 3.8 + urllib3 commands = pytest {posargs}