Merge pull request #991 from YunoHost/ci-build-&-install-deb

[enh] CI V2

.gitlab-ci.yml

@@ -1,142 +1,22 @@
 stages:
-  - postinstall
+  - build
+  - install
   - tests
   - lint
+  - doc
 
-########################################
+default:
-# POSTINSTALL
+  tags:
-########################################
+    - yunohost-ci
+  # All jobs are interruptible by default
+  interruptible: true
 
-postinstall:
+variables:
-  image: before-postinstall
+    YNH_BUILD_DIR: "ynh-build"
-  stage: postinstall
-  script:
-    - apt install --no-install-recommends -y $(cat debian/control | grep "^Depends" -A50 | grep "Recommends:" -B50 | grep "^ *," | grep -o -P "[\w\-]{3,}")
-    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns
 
-########################################
+include:
-# TESTS
+  - local: .gitlab/ci/build.gitlab-ci.yml
-########################################
+  - local: .gitlab/ci/install.gitlab-ci.yml
-
+  - local: .gitlab/ci/test.gitlab-ci.yml
-.test-stage:
+  - local: .gitlab/ci/lint.gitlab-ci.yml
-  image: after-postinstall
+  - local: .gitlab/ci/doc.gitlab-ci.yml
-  stage: tests
-  before_script:
-    - apt-get install python-pip -y
-    - mkdir -p .pip
-    - pip install -U pip
-    - hash -d pip
-    - pip --cache-dir=.pip install pytest pytest-sugar pytest-mock requests-mock mock
-    - export PYTEST_ADDOPTS="--color=yes"
-  cache:
-    paths: 
-      - .pip
-      - src/yunohost/tests/apps
-    key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
-
-root-tests:
-  extends: .test-stage
-  script:
-    - py.test tests
-
-test-apps:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_apps.py
-
-test-appscatalog:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_appscatalog.py
-
-test-appurl:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_appurl.py
-
-test-apps-arguments-parsing:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_apps_arguments_parsing.py 
-
-test-backuprestore:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_backuprestore.py
-
-test-changeurl:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_changeurl.py
-
-test-permission:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_permission.py
-
-test-settings:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_settings.py
-
-test-user-group:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_user-group.py
-    
-test-regenconf:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_regenconf.py
-
-test-service:
-  extends: .test-stage
-  script:
-    - cd src/yunohost
-    - py.test tests/test_service.py
-
-########################################
-# LINTER
-########################################
-
-.lint-stage:
-  image: before-postinstall
-  stage: lint
-  before_script:
-    - apt-get install python-pip -y
-    - mkdir -p .pip
-    - pip install -U pip
-    - hash -d pip
-    - pip --cache-dir=.pip install tox
-  cache:
-    paths: 
-      - .pip
-      - .tox
-    key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
-
-lint:
-  extends: .lint-stage
-  allow_failure: true
-  script:
-    - tox -e lint
-
-invalidcode:
-  extends: .lint-stage
-  script:
-    - tox -e invalidcode
-
-# Disabled, waiting for buster
-#format-check:
-#  extends: .lint-stage
-#  script:
-#    - black --check --diff

.gitlab/ci/build.gitlab-ci.yml

@@ -0,0 +1,52 @@
+.build-stage:
+  stage: build
+  image: "before-install"
+  variables:
+    YNH_SOURCE: "https://github.com/yunohost"
+  before_script:
+    - mkdir -p $YNH_BUILD_DIR
+  artifacts:
+    paths:
+      - $YNH_BUILD_DIR/*.deb
+
+.build_script: &build_script
+  - cd $YNH_BUILD_DIR/$PACKAGE
+  - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null)
+  - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)"
+  - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build."
+  - debuild --no-lintian -us -uc
+
+########################################
+# BUILD DEB
+########################################
+
+build-yunohost:
+  extends: .build-stage
+  variables:
+    PACKAGE: "yunohost"
+  script:
+    - git ls-files | xargs tar -czf archive.tar.gz
+    - mkdir -p $YNH_BUILD_DIR/$PACKAGE
+    - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE
+    - rm archive.tar.gz
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script
+
+
+build-ssowat:
+  extends: .build-stage
+  variables:
+    PACKAGE: "ssowat"
+  script:
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script
+
+build-moulinette:
+  extends: .build-stage
+  variables:
+    PACKAGE: "moulinette"
+  script:
+    - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $(pwd)/$YNH_BUILD_DIR/$PACKAGE
+    - *build_script

.gitlab/ci/doc.gitlab-ci.yml

@@ -0,0 +1,14 @@
+########################################
+# DOC
+########################################
+
+generate-helpers-doc:
+  stage: doc
+  image: "before-install"
+  needs: []
+  script:
+    - cd doc
+    - python generate_helper_doc.py
+  artifacts:
+    paths:
+      - doc/helpers.html

.gitlab/ci/install.gitlab-ci.yml

@@ -0,0 +1,27 @@
+.install-stage:
+  stage: install
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+
+########################################
+# INSTALL DEB
+########################################
+
+upgrade:
+  extends: .install-stage
+  image: "after-install"
+  script:
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+
+
+install-postinstall:
+  extends: .install-stage
+  image: "before-install"
+  script:
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns

.gitlab/ci/lint.gitlab-ci.yml

@@ -0,0 +1,24 @@
+########################################
+# LINTER
+########################################
+
+lint:
+  stage: lint
+  image: "before-install"
+  needs: []
+  allow_failure: true
+  script:
+    - tox -e lint
+
+invalidcode:
+  stage: lint
+  image: "before-install"
+  needs: []
+  script:
+    - tox -e invalidcode
+
+# Disabled, waiting for buster
+#format-check:
+#  extends: .lint-stage
+#  script:
+#    - black --check --diff

.gitlab/ci/test.gitlab-ci.yml

@@ -0,0 +1,119 @@
+.install_debs: &install_debs
+    - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ./$YNH_BUILD_DIR/*.deb
+
+.test-stage:
+  stage: tests
+  image: "after-install"
+  variables:
+    PYTEST_ADDOPTS: "--color=yes"
+  before_script:
+    - *install_debs
+  cache:
+    paths: 
+      - src/yunohost/tests/apps
+    key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+      - job: upgrade
+
+
+########################################
+# TESTS
+########################################
+
+full-tests:
+  stage: tests
+  image: "before-install"
+  variables:
+    PYTEST_ADDOPTS: "--color=yes"
+  before_script:
+    - *install_debs
+    - yunohost tools postinstall -d domain.tld -p the_password --ignore-dyndns
+  script:
+    - pytest --cov=yunohost tests/ src/yunohost/tests/ --junitxml=report.xml
+  needs:
+      - job: build-yunohost
+        artifacts: true
+      - job: build-ssowat
+        artifacts: true
+      - job: build-moulinette
+        artifacts: true
+  artifacts:
+    reports:
+      junit: report.xml
+
+root-tests:
+  extends: .test-stage
+  script:
+    - py.test tests
+
+test-apps:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_apps.py
+
+test-appscatalog:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_appscatalog.py
+
+test-appurl:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_appurl.py
+
+test-apps-arguments-parsing:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_apps_arguments_parsing.py
+
+test-backuprestore:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_backuprestore.py
+
+test-changeurl:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_changeurl.py
+
+test-permission:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_permission.py
+
+test-settings:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_settings.py
+
+test-user-group:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_user-group.py
+    
+test-regenconf:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_regenconf.py
+
+test-service:
+  extends: .test-stage
+  script:
+    - cd src/yunohost
+    - py.test tests/test_service.py

pytest.ini

@@ -10,3 +10,5 @@ markers =
     with_legacy_app_installed
     with_backup_recommended_app_installed_with_ynh_restore
     with_permission_app_installed
+filterwarnings =
+    ignore::urllib3.exceptions.InsecureRequestWarning

src/yunohost/tests/conftest.py

@@ -19,6 +19,9 @@ def clone_test_app(request):
     else:
         os.system("cd %s/apps && git pull > /dev/null 2>&1" % cwd)
 
+def get_test_apps_dir():
+    cwd = os.path.split(os.path.realpath(__file__))[0]
+    return os.path.join(cwd, "apps")
 
 @contextmanager
 def message(mocker, key, **kwargs):

src/yunohost/tests/test_apps.py

@@ -4,7 +4,7 @@ import pytest
 import shutil
 import requests
 
-from conftest import message, raiseYunohostError
+from conftest import message, raiseYunohostError, get_test_apps_dir
 
 from moulinette.utils.filesystem import mkdir
 
@@ -150,7 +150,7 @@ def app_is_exposed_on_http(domain, path, message_in_page):
 def install_legacy_app(domain, path, public=True):
 
     app_install(
-        "./tests/apps/legacy_app_ynh",
+        os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
         args="domain=%s&path=%s&is_public=%s" % (domain, path, 1 if public else 0),
         force=True,
     )
@@ -159,14 +159,14 @@ def install_legacy_app(domain, path, public=True):
 def install_full_domain_app(domain):
 
     app_install(
-        "./tests/apps/full_domain_app_ynh", args="domain=%s" % domain, force=True
+        os.path.join(get_test_apps_dir(), "full_domain_app_ynh"), args="domain=%s" % domain, force=True
     )
 
 
 def install_break_yo_system(domain, breakwhat):
 
     app_install(
-        "./tests/apps/break_yo_system_ynh",
+        os.path.join(get_test_apps_dir(), "break_yo_system_ynh"),
         args="domain=%s&breakwhat=%s" % (domain, breakwhat),
         force=True,
     )
@@ -376,7 +376,7 @@ def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain):
 
     with pytest.raises(YunohostError):
         with message(mocker, "app_action_broke_system"):
-            app_upgrade("break_yo_system", file="./tests/apps/break_yo_system_ynh")
+            app_upgrade("break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh"))
 
 
 def test_failed_multiple_app_upgrade(mocker, secondary_domain):
@@ -389,7 +389,7 @@ def test_failed_multiple_app_upgrade(mocker, secondary_domain):
             app_upgrade(
                 ["break_yo_system", "legacy_app"],
                 file={
-                    "break_yo_system": "./tests/apps/break_yo_system_ynh",
+                    "break_yo_system": os.path.join(get_test_apps_dir(), "break_yo_system_ynh"),
-                    "legacy": "./tests/apps/legacy_app_ynh",
+                    "legacy": os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
                 },
             )

src/yunohost/tests/test_appurl.py

@@ -1,4 +1,7 @@
 import pytest
+import os
+
+from conftest import get_test_apps_dir
 
 from yunohost.utils.error import YunohostError
 from yunohost.app import app_install, app_remove
@@ -43,19 +46,19 @@ def test_urlavailable():
 
 def test_registerurl():
 
-    app_install("./tests/apps/register_url_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"),
                 args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"), force=True)
 
     assert not domain_url_available(maindomain, "/urlregisterapp")
 
     # Try installing at same location
     with pytest.raises(YunohostError):
-        app_install("./tests/apps/register_url_app_ynh",
+        app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"),
                     args="domain=%s&path=%s" % (maindomain, "/urlregisterapp"), force=True)
 
 
 def test_registerurl_baddomain():
 
     with pytest.raises(YunohostError):
-        app_install("./tests/apps/register_url_app_ynh",
+        app_install(os.path.join(get_test_apps_dir(), "register_url_app_ynh"),
                     args="domain=%s&path=%s" % ("yolo.swag", "/urlregisterapp"), force=True)

src/yunohost/tests/test_backuprestore.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import subprocess
 
-from conftest import message, raiseYunohostError
+from conftest import message, raiseYunohostError, get_test_apps_dir
 
 from yunohost.app import app_install, app_remove, app_ssowatconf
 from yunohost.app import _is_installed
@@ -117,9 +117,9 @@ def app_is_installed(app):
 def backup_test_dependencies_are_met():
 
     # Dummy test apps (or backup archives)
-    assert os.path.exists("./tests/apps/backup_wordpress_from_2p4")
+    assert os.path.exists(os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4"))
-    assert os.path.exists("./tests/apps/legacy_app_ynh")
+    assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh"))
-    assert os.path.exists("./tests/apps/backup_recommended_app_ynh")
+    assert os.path.exists(os.path.join(get_test_apps_dir(), "backup_recommended_app_ynh"))
 
     return True
 
@@ -175,7 +175,7 @@ def uninstall_test_apps_if_needed():
 
 def install_app(app, path, additionnal_args=""):
 
-    app_install("./tests/apps/%s" % app,
+    app_install(os.path.join(get_test_apps_dir(), app),
                 args="domain=%s&path=%s%s" % (maindomain, path,
                                               additionnal_args), force=True)
 
@@ -184,22 +184,22 @@ def add_archive_wordpress_from_2p4():
 
     os.system("mkdir -p /home/yunohost.backup/archives")
 
-    os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.info.json \
+    os.system("cp " + os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4/backup.info.json") + \
-               /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json")
+               " /home/yunohost.backup/archives/backup_wordpress_from_2p4.info.json")
 
-    os.system("cp ./tests/apps/backup_wordpress_from_2p4/backup.tar.gz \
+    os.system("cp " + os.path.join(get_test_apps_dir(), "backup_wordpress_from_2p4/backup.tar.gz") + \
-               /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz")
+               " /home/yunohost.backup/archives/backup_wordpress_from_2p4.tar.gz")
 
 
 def add_archive_system_from_2p4():
 
     os.system("mkdir -p /home/yunohost.backup/archives")
 
-    os.system("cp ./tests/apps/backup_system_from_2p4/backup.info.json \
+    os.system("cp " + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.info.json") + \
-               /home/yunohost.backup/archives/backup_system_from_2p4.info.json")
+               " /home/yunohost.backup/archives/backup_system_from_2p4.info.json")
 
-    os.system("cp ./tests/apps/backup_system_from_2p4/backup.tar.gz \
+    os.system("cp " + os.path.join(get_test_apps_dir(), "backup_system_from_2p4/backup.tar.gz") + \
-               /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz")
+               " /home/yunohost.backup/archives/backup_system_from_2p4.tar.gz")
 
 #
 # System backup                                                              #

src/yunohost/tests/test_changeurl.py

@@ -1,6 +1,9 @@
 import pytest
 import time
 import requests
+import os
+
+from conftest import get_test_apps_dir
 
 from yunohost.app import app_install, app_change_url, app_remove, app_map
 from yunohost.domain import _get_maindomain
@@ -20,7 +23,7 @@ def teardown_function(function):
 
 
 def install_changeurl_app(path):
-    app_install("./tests/apps/change_url_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "change_url_app_ynh"),
                 args="domain=%s&path=%s" % (maindomain, path), force=True)
 
 

src/yunohost/tests/test_permission.py

@@ -1,7 +1,8 @@
 import requests
 import pytest
+import os
 
-from conftest import message, raiseYunohostError
+from conftest import message, raiseYunohostError, get_test_apps_dir
 
 from yunohost.app import app_install, app_remove, app_change_url, app_list, app_map, _installed_apps
 from yunohost.user import user_list, user_create, user_delete, \
@@ -418,7 +419,7 @@ def test_permission_remove_url():
 
 
 def test_permission_app_install():
-    app_install("./tests/apps/permissions_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
                 args="domain=%s&path=%s&is_public=0&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
 
     res = user_permission_list(full=True)['permissions']
@@ -446,7 +447,7 @@ def test_permission_app_install():
 
 
 def test_permission_app_remove():
-    app_install("./tests/apps/permissions_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
                 args="domain=%s&path=%s&is_public=0&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
     app_remove("permissions_app")
 
@@ -456,7 +457,7 @@ def test_permission_app_remove():
 
 
 def test_permission_app_change_url():
-    app_install("./tests/apps/permissions_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
                 args="domain=%s&path=%s&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
 
     # FIXME : should rework this test to look for differences in the generated app map / app tiles ...
@@ -475,7 +476,7 @@ def test_permission_app_change_url():
 
 def test_permission_app_propagation_on_ssowat():
 
-    app_install("./tests/apps/permissions_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
                 args="domain=%s&path=%s&is_public=1&admin=%s" % (maindomain, "/urlpermissionapp", "alice"), force=True)
 
     res = user_permission_list(full=True)['permissions']
@@ -505,7 +506,7 @@ def test_permission_app_propagation_on_ssowat():
 
 def test_permission_legacy_app_propagation_on_ssowat():
 
-    app_install("./tests/apps/legacy_app_ynh",
+    app_install(os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
                 args="domain=%s&path=%s" % (maindomain, "/legacy"), force=True)
 
     # App is configured as public by default using the legacy unprotected_uri mechanics

tox.ini

@@ -10,6 +10,7 @@ deps =
   pytest >= 4.6.3, < 5.0
   pyyaml >= 5.1.2, < 6.0
   flake8 >= 3.7.9, < 3.8
+  urllib3
 commands =
     pytest {posargs}