diff --git a/README.md b/README.md index d7344c0..d9e15eb 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Please report issues on the following repository: * [4. Testing the web interface](#4-testing-the-web-interface) * [Advanced: using snapshots](#advanced-using-snapshots) * [Alternative: Only Virtualbox](#alternative-using-only-virtualbox) + * [Troubleshooting](#troubleshooting) - [Remote Development Environment](#remote-development-environment) * [1. Setup your VPS and install YunoHost](#1-setup-your-vps-and-install-yunohost) @@ -138,6 +139,22 @@ answering the default (just pressing enter) to all questions is fine. sudo lxd init ``` +Pre-built images are centralized on `devbaseimgs.yunohost.org` and we'll download them from there to speed things up: + +```bash +sudo lxc remote add yunohost https://devbaseimgs.yunohost.org --public +``` + +On Archlinux-based distributions (Arch, Manjaro, ...) it was found that it's needed +that LXC/LXD will throw some error about "newuidmap failed to write mapping / Failed +to set up id mapping" ... It can be [fixed with the following](https://discuss.linuxcontainers.org/t/solved-arch-linux-containers-only-run-when-security-privileged-true/4006/4) : + +``` +# N.B.: this is ONLY for Arch-based distros +$ echo "root:1000000:65536" > /etc/subuid +$ echo "root:1000000:65536" > /etc/subgid +``` + Then, go into your favorite development folder and deploy `ynh-dev` with: ```bash @@ -154,34 +171,15 @@ between the host and the LXC. When ran on the host, the `./ynh-dev` command allows you to manage YunoHost's dev LXCs. -First, you might want to build the base LXC with: +Start your actual dev LXC using : ```bash $ cd ynh-dev # if not already done -$ ./ynh-dev rebuild -# ... This will take some time, grab your favorite beverage ... -``` - -This should create a fresh Debian Stretch LXC, install Yunohost inside and save -the result as `ynh-dev-base` which can then be used to create your actual dev -LXC. (This base can then be used to recreate a fresh Yunohost LXC if you need to -destroy your work LXC) - -On Archlinux-based distributions (Arch, Manjaro, ...) it was found that it's needed -that LXC/LXD will throw some error about "newuidmap failed to write mapping / Failed -to set up id mapping" ... It can be [fixed with the following](https://discuss.linuxcontainers.org/t/solved-arch-linux-containers-only-run-when-security-privileged-true/4006/4) : - -``` -$ echo "root:1000000:65536" > /etc/subuid -$ echo "root:1000000:65536" > /etc/subgid -``` - -Then start your actual dev LXC using : - -```bash $ ./ynh-dev start ``` +This should automatically download from `devbaseimgs.yunohost.org` a pre-build ynh-dev LXC image running Yunohost unstable, and create a fresh container from it. + After starting the LXC, your terminal will automatically be attached to it. If you later disconnect from the LXC, you can go back in with `./ynh-dev attach`. Later, you might want to destroy the LXC. You can do so with `./ynh-dev destroy`. ## 3. Development and container testing @@ -225,6 +223,13 @@ this repository. This is a known working setup used by some developers. Please see the ["virtualbox" branch](https://github.com/YunoHost/ynh-dev/tree/virtualbox#develop-on-your-local-machine) for more. +## Troubleshooting +If you experiment network issues with your lxd during rebuild container steps. Probably your container are not able to get a local IP with DHCP. + +It could be due to bridge conflict (for example if you have lxc installed too) or dnsmasq port already used. + +This [ticket](https://github.com/YunoHost/issues/issues/1664) could help. + # Remote Development Environment Here is the development flow: diff --git a/ynh-dev b/ynh-dev index e7ac21b..e743467 100755 --- a/ynh-dev +++ b/ynh-dev @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash function show_usage() { cat </dev/null then - sudo lxc image info $BOX-base &>/dev/null || critical "You should first build the base YunoHost LXC using ./ynh-dev rebuild" - set -eu - set -x - sudo lxc launch $BOX-base $BOX - sudo lxc config set $BOX security.privileged true - sudo lxc config set $BOX security.nesting true # Need this for apparmor for some reason + if ! sudo lxc image info $BOX-base &>/dev/null + then + LXC_BASE="ynh-dev-$DIST-amd64-$YNH_BRANCH-base" + sudo lxc launch yunohost:$LXC_BASE $BOX -c security.nesting=true -c security.privileged=true \ + || critical "Failed to launch the container ?" + else + sudo lxc launch $BOX-base $BOX -c security.nesting=true -c security.privileged=true + fi sudo lxc config device add $BOX ynhdev-shared-folder disk path=/ynh-dev source="$(readlink -f $(pwd))" - set +x info "Now attaching to the container" else info "Attaching to existing container" fi - attach_ynhdev $BOX + attach_ynhdev "$@" } function attach_ynhdev() { - # We need those link for the yunohost-admin build to work without r/w - # permission in the shared folder. - # - # The target folders (/var/cache/ynh-dev/...) are then created during the - # './ynh-dev use-git yunohost-admin' - if [[ ! -L ./yunohost-admin/src/dist ]] - then - ln -s /var/cache/ynh-dev/yunohost-admin/node_modules ./yunohost-admin/src/ - ln -s /var/cache/ynh-dev/yunohost-admin/dist ./yunohost-admin/src/ - fi - check_lxd_setup - local BOX=${1:-ynh-dev-buster} + local DIST=${1:-buster} + local YNH_BRANCH=${3:-unstable} + local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} sudo lxc start $BOX 2>/dev/null || true sudo lxc exec $BOX --cwd /ynh-dev -- /bin/bash } @@ -176,7 +169,8 @@ function destroy_ynhdev() { check_lxd_setup local DIST=${1:-buster} - local BOX=${2:-ynh-dev}-${DIST} + local YNH_BRANCH=${3:-unstable} + local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} sudo lxc stop $BOX sudo lxc delete $BOX } @@ -186,18 +180,16 @@ function rebuild_ynhdev() check_lxd_setup local DIST=${1:-buster} - local BOX=${2:-ynh-dev}-${DIST} + local YNH_BRANCH=${3:-unstable} + local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} set -x sudo lxc info $BOX-rebuild >/dev/null && sudo lxc delete $BOX-rebuild --force - sudo lxc launch images:debian/$DIST/amd64 $BOX-rebuild - sudo lxc config set $BOX-rebuild security.privileged true - sudo lxc config set $BOX-rebuild security.nesting true # Need this for apparmor for some reason - sudo lxc restart $BOX-rebuild + sudo lxc launch images:debian/$DIST/amd64 $BOX-rebuild -c security.nesting=true -c security.privileged=true sleep 5 sudo lxc exec $BOX-rebuild -- apt install curl -y - INSTALL_SCRIPT="https://install.yunohost.org" - sudo lxc exec $BOX-rebuild -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d unstable" + INSTALL_SCRIPT="https://install.yunohost.org/$DIST" + sudo lxc exec $BOX-rebuild -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH" sudo lxc stop $BOX-rebuild sudo lxc publish $BOX-rebuild --alias $BOX-base set +x @@ -219,7 +211,7 @@ function use_git() PACKAGES=('moulinette' 'ssowat' 'yunohost' 'yunohost-admin') fi - for i in ${!PACKAGES[@]}; + for i in "${!PACKAGES[@]}"; do case ${PACKAGES[i]} in ssowat) @@ -228,7 +220,7 @@ function use_git() ;; moulinette) create_sym_link "/ynh-dev/moulinette/locales" "/usr/share/moulinette/locale" - create_sym_link "/ynh-dev/moulinette/moulinette" "/usr/lib/python2.7/dist-packages/moulinette" + create_sym_link "/ynh-dev/moulinette/moulinette" "/usr/lib/python3/dist-packages/moulinette" success "Now using Git repository for Moulinette" ;; yunohost) @@ -241,6 +233,7 @@ function use_git() create_sym_link "/ynh-dev/yunohost/sbin/yunohost-reset-ldap-password" "/usr/sbin/yunohost-reset-ldap-password" # data + python3 "/ynh-dev/yunohost/data/actionsmap/yunohost_completion.py" create_sym_link "/ynh-dev/yunohost/data/bash-completion.d/yunohost" "/etc/bash_completion.d/yunohost" create_sym_link "/ynh-dev/yunohost/data/actionsmap/yunohost.yml" "/usr/share/moulinette/actionsmap/yunohost.yml" create_sym_link "/ynh-dev/yunohost/data/hooks" "/usr/share/yunohost/hooks" @@ -249,9 +242,6 @@ function use_git() create_sym_link "/ynh-dev/yunohost/data/helpers.d" "/usr/share/yunohost/helpers.d" create_sym_link "/ynh-dev/yunohost/data/other" "/usr/share/yunohost/yunohost-config/moulinette" - # debian - create_sym_link "/ynh-dev/yunohost/debian/conf/pam/mkhomedir" "/usr/share/pam-configs/mkhomedir" - # lib create_sym_link "/ynh-dev/yunohost/lib/metronome/modules/ldap.lib.lua" "/usr/lib/metronome/modules/ldap.lib.lua" create_sym_link "/ynh-dev/yunohost/lib/metronome/modules/mod_auth_ldap2.lua" "/usr/lib/metronome/modules/mod_auth_ldap2.lua" @@ -270,52 +260,80 @@ function use_git() ;; yunohost-admin) - getent passwd ynhdev > /dev/null - if [ $? -eq 2 ]; then - useradd ynhdev - fi - + rm -r /var/cache/ynh-dev/yunohost-admin mkdir -p /var/cache/ynh-dev/yunohost-admin/ - mkdir -p /var/cache/ynh-dev/yunohost-admin/dist - chown -R ynhdev /var/cache/ynh-dev/yunohost-admin/ - create_sym_link "/ynh-dev/yunohost-admin/src/package.json" "/var/cache/ynh-dev/yunohost-admin/package.json" - create_sym_link "/ynh-dev/yunohost-admin/src/package-lock.json" "/var/cache/ynh-dev/yunohost-admin/package-lock.json" - create_sym_link "/ynh-dev/yunohost-admin/src" "/usr/share/yunohost/admin" + create_sym_link "/ynh-dev/yunohost-admin/app/.env" "/var/cache/ynh-dev/yunohost-admin/.env" + create_sym_link "/var/cache/ynh-dev/yunohost-admin/node_modules" "/ynh-dev/yunohost-admin/app/node_modules" + create_sym_link "/ynh-dev/yunohost-admin/app/package.json" "/var/cache/ynh-dev/yunohost-admin/package.json" + create_sym_link "/ynh-dev/yunohost-admin/app/package-lock.json" "/var/cache/ynh-dev/yunohost-admin/package-lock.json" + mv "/etc/nginx/conf.d/yunohost_admin.conf.inc" "/etc/nginx/conf.d/yunohost_admin.conf.inc.bkp" + trap 'on_exit' exit + on_exit() + { + mv "/etc/nginx/conf.d/yunohost_admin.conf.inc.bkp" "/etc/nginx/conf.d/yunohost_admin.conf.inc" + systemctl reload nginx + } - if [ ! -e /ynh-dev/yunohost-admin/src/dist ] + cd /var/cache/ynh-dev/yunohost-admin/ + # Create .env file with the vm ip + # Will be used by webpack-dev-server to proxy api requests. + if [[ ! -e .env ]] then - echo "If npm install fails to create the dist and node_modules folder, maybe you need to run the following *in the host!* : " - echo " " - echo "cd /your/dev/env/yunohost-admin/src/" - echo "ln -s /var/cache/ynh-dev/yunohost-admin/dist ./dist" - echo "ln -s /var/cache/ynh-dev/yunohost-admin/node_modules ./node_modules" - echo " " + info "Creating .env file" + IP=$(hostname -I | tr ' ' '\n' | grep "\.") + echo "VUE_APP_IP=$IP" > .env fi - # Install npm dependencies if needed - cd /var/cache/ynh-dev/yunohost-admin/ - if [ ! -e node_modules/gulp/bin/gulp.js ] + # Allow port 8080 in config file or else the dev server will stop working after postinstall + if [[ ! -e /etc/yunohost/installed ]] + then + python3 - < /etc/nginx/conf.d/yunohost_admin.conf.inc +location /yunohost/admin { + proxy_pass https://localhost:8080/yunohost/admin; + proxy_redirect off; + proxy_set_header Host \$host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host \$server_name; + proxy_set_header X-Forwarded-Port \$server_port; - cd /ynh-dev/yunohost-admin/src - su ynhdev -c "./node_modules/gulp/bin/gulp.js build --dev" - warn "-------------------------------------------------------- " - warn "Launching gulp ... " - warn "NB : This command will keep running and watch for changes" - warn " in the folder /ynh-dev/yunohost-admin/src, such that you" - warn "don't need to re-run npm yourself everytime you change " - warn "something ! " - warn "-------------------------------------------------------- " - su ynhdev -c "./node_modules/gulp/bin/gulp.js watch --dev" + proxy_http_version 1.1; + proxy_set_header Upgrade \$http_upgrade; + proxy_set_header Connection "upgrade"; +} +EOF + systemctl reload nginx + cd /ynh-dev/yunohost-admin/app/ + npm run serve + ;; + *) + error "Invalid package '${PACKAGES[i]}': correct arguments are 'yunohost', 'ssowat', 'moulinette', 'yunohost-admin' or nothing for all" ;; *) error "Invalid package '${PACKAGES[i]}': correct arguments are 'yunohost', 'ssowat', 'moulinette', 'yunohost-admin' or nothing for all" @@ -328,7 +346,7 @@ function run_tests() { assert_inside_vm local PACKAGES="$@" - for PACKAGE in "$PACKAGES"; + for PACKAGE in $PACKAGES; do TEST_FUNCTION=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $3}') TEST_MODULE=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $2}') @@ -339,14 +357,14 @@ function run_tests() # Pytest and tests dependencies if ! type "pytest" > /dev/null 2>&1; then info "> Installing pytest ..." - apt-get install python-pip -y - pip2 install pytest pytest-sugar + apt-get install python3-pip -y + pip3 install pytest pytest-sugar fi for DEP in "pytest-mock requests-mock mock" do - if [ -z "$(pip show $DEP)" ]; then - info "Installing $DEP with pip" - pip2 install $DEP + if [ -z "$(pip3 show $DEP)" ]; then + info "Installing $DEP with pip3" + pip3 install $DEP fi done