From b03679c70da584a42dd5bd9f8fb213b070c47c69 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Sun, 20 May 2018 23:57:36 +0200 Subject: [PATCH] Add redis, worker and admin --- conf/pagure-worker.service | 14 ++++++++++++ conf/pagure.cfg.sample | 6 ++--- scripts/_common.sh | 45 ++++++++++++++++++++++++++++++++++++++ scripts/install | 16 ++++++++++++-- scripts/remove | 17 ++++++++++++++ 5 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 conf/pagure-worker.service diff --git a/conf/pagure-worker.service b/conf/pagure-worker.service new file mode 100644 index 0000000..f59f43e --- /dev/null +++ b/conf/pagure-worker.service @@ -0,0 +1,14 @@ +[Unit] +Description=__APP__ worker for backend git interaction +After=redis.target + +[Service] +ExecStart=/usr/bin/celery worker -A pagure.lib.tasks --loglevel=info +Environment="PAGURE_CONFIG=__FINALPATH__/pagure.cfg" +Type=simple +User=__APP__ +Group=__APP__ +Restart=on-failure + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/pagure.cfg.sample b/conf/pagure.cfg.sample index e5cc834..449ca5e 100644 --- a/conf/pagure.cfg.sample +++ b/conf/pagure.cfg.sample @@ -24,7 +24,7 @@ DB_URL = 'postgres://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__' ADMIN_GROUP = ['sysadmin-main'] ### Hard-coded list of global admins -PAGURE_ADMIN_USERS = [] +PAGURE_ADMIN_USERS = ['__ADMIN__'] ### The email address to which the flask.log will send the errors (tracebacks) EMAIL_ERROR = 'root@__DOMAIN__' @@ -153,9 +153,9 @@ WEBHOOK = False ### Redis configuration # A redis server is required for both the Event-Source server or the web-hook # server. -REDIS_HOST = '0.0.0.0' +REDIS_HOST = '127.0.0.1' REDIS_PORT = 6379 -REDIS_DB = 0 +REDIS_DB = __REDIS_DB__ # Authentication related configuration option diff --git a/scripts/_common.sh b/scripts/_common.sh index ccac9d5..ae2949f 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,5 +1,50 @@ #!/bin/bash +#================================================= +# +# Redis HELPERS +# +# Point of contact : Jean-Baptiste Holcroft +#================================================= + +# get the first available redis database +# +# usage: ynh_redis_get_free_db +# | returns: the database number to use +ynh_redis_get_free_db() { + local result max db + result=$(redis-cli INFO keyspace) + + # get the num + max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+") + + db=0 + # default Debian setting is 15 databases + for i in $(seq 0 "$max") + do + if ! echo "$result" | grep -q "db$i" + then + db=$i + break 1 + db=-1 + fi + done + + test "$db" -eq -1 && ynh_die "No available Redis databases..." + + echo "$db" +} + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_redis_remove_db database +# | arg: database - the database to erase +ynh_redis_remove_db() { + local db=$1 + redis-cli -n "$db" flushall +} + ynh_check_global_uwsgi_config () { uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" diff --git a/scripts/install b/scripts/install index 2652cc8..0627461 100755 --- a/scripts/install +++ b/scripts/install @@ -75,7 +75,7 @@ ynh_app_setting_set "$app" final_path "$final_path" ynh_install_app_dependencies git virtualenv python-virtualenv libgit2-dev \ libjpeg-dev gcc libffi-dev python-dev python-cffi \ python-gdbm python-psycopg2 \ - postgresql uwsgi uwsgi-plugin-python + postgresql uwsgi uwsgi-plugin-python redis-server #================================================= # CREATE A PostgreSQL DATABASE @@ -106,6 +106,13 @@ ynh_system_user_create "$app" "${final_path}" #================================================= # SPECIFIC SETUP +#================================================= +# SETUP SYSTEMD +#================================================= + +# Create a dedicated systemd config +ynh_add_systemd_config "$app-worker" "pagure-worker.service" + #================================================= # setup pagure.cfg #================================================= @@ -113,13 +120,18 @@ ynh_system_user_create "$app" "${final_path}" secret_key=$(ynh_string_random) salt_email=$(ynh_string_random) +redis_db=$(ynh_redis_get_free_db) +ynh_app_setting_set "$app" redis_db "$redis_db" + cp ../conf/pagure.cfg.sample "$final_path/pagure.cfg" +ynh_replace_string "__REDIS_DB__" "$redis_db" "$final_path/pagure.cfg" ynh_replace_string "__SECRET_KEY__" "$secret_key" "$final_path/pagure.cfg" ynh_replace_string "__DB_USER__" "$app" "$final_path/pagure.cfg" ynh_replace_string "__DB_PWD__" "$db_pwd" "$final_path/pagure.cfg" ynh_replace_string "__DB_NAME__" "$db_name" "$final_path/pagure.cfg" ynh_replace_string "__DOMAIN__" "$domain" "$final_path/pagure.cfg" ynh_replace_string "__SALT_EMAIL__" "$salt_email" "$final_path/pagure.cfg" +ynh_replace_string "__ADMIN__" "$admin" "$final_path/pagure.cfg" #================================================= # setup pagure.wsgi @@ -153,7 +165,7 @@ virtualenv "${final_path}/venv" pip install cffi pip install pygit2==0.24 pip install -r "${final_path}/pagure/requirements.txt" - pip install psycopg2 cryptography py-bcrypt python-fedora + pip install psycopg2-binary cryptography py-bcrypt python-fedora ) #================================================= diff --git a/scripts/remove b/scripts/remove index c2939d5..d19575b 100755 --- a/scripts/remove +++ b/scripts/remove @@ -17,6 +17,16 @@ app=$YNH_APP_INSTANCE_NAME db_name=$(ynh_app_setting_get "$app" db_name) domain=$(ynh_app_setting_get "$app" domain) +redis_db=$(ynh_app_setting_get "$app" redis_db) + +#================================================= +# STANDARD REMOVE +#================================================= +# STOP AND REMOVE SERVICE +#================================================= + +# Remove the dedicated systemd config +ynh_remove_systemd_config "$app-worker" #================================================= # REMOVE uwsgi and systemd files @@ -31,6 +41,13 @@ ynh_remove_uwsgi_service # Remove a database if it exists, along with the associated user ynh_psql_remove_db "$db_name" "$app" +#================================================= +# REMOVE THE Redis DATABASE +#================================================= + +# Remove a database if it exists, along with the associated user +ynh_redis_remove_db "$redis_db" + #================================================= # REMOVE DEPENDENCIES #=================================================