From 53fbf3a9baa67d9a1655f5ba9bf1009c5ce98e0f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 23 Nov 2017 22:56:00 +0100 Subject: [PATCH] First version with package_check saying ok for install and remove --- README.md | 8 +- conf/app.src | 4 +- conf/nginx.conf | 15 +++ conf/pagure.cfg.sample | 210 ++++++++++++++++++++++++++++++++++++++++ conf/pagure.wsgi | 28 ++++++ conf/systemd.service | 13 --- conf/uwsgi-app@.service | 16 +++ conf/uwsgi-app@.socket | 11 +++ conf/uwsgi.ini | 12 +++ manifest.json | 6 +- scripts/_common.sh | 77 +++++++++++++++ scripts/install | 77 ++++++++++----- scripts/remove | 27 +++--- 13 files changed, 446 insertions(+), 58 deletions(-) create mode 100644 conf/nginx.conf create mode 100644 conf/pagure.cfg.sample create mode 100644 conf/pagure.wsgi delete mode 100644 conf/systemd.service create mode 100644 conf/uwsgi-app@.service create mode 100644 conf/uwsgi-app@.socket create mode 100644 conf/uwsgi.ini diff --git a/README.md b/README.md index 946273b..262aa75 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ # pagure_ynh Pagure, a git centered forge for YunoHost -Due to the usage of RemoteCollection, Pagure can't be use on Debian 8. +Due to the usage of RemoteCollection, Pagure can't be use on Debian 8, please use Debian 9 branch of YunoHost: https://github.com/YunoHost/yunohost/tree/stretch -It needs a libgit2-dev>=0.22: -https://github.com/libgit2/pygit2/blob/62c70e852da23bcb60e64996f6326a3e2a800469/CHANGELOG.rst#0220-2015-01-16 - -Stretch has 0.25, but it will wait for YunoHost to be compatible: -https://packages.debian.org/search?keywords=libgit2&searchon=names&suite=all§ion=all +This package was only tested with package_check + VM, _do not try to install it on a production system_ without knowing what you do. diff --git a/conf/app.src b/conf/app.src index 0c76cec..6376669 100644 --- a/conf/app.src +++ b/conf/app.src @@ -1,5 +1,5 @@ -SOURCE_URL=https://releases.pagure.org/pagure/pagure-3.8.tar.gz -SOURCE_SUM=ba53d32cac0acb4dc111accb14639939c06927d2cb96245c38ddf631b7d82a4e +SOURCE_URL=https://releases.pagure.org/pagure/pagure-3.10.1.tar.gz +SOURCE_SUM=853a7e4bf84305a2695b8608984cd9e199e19693507ddf6cdef4ec7fc4e2be25 SOURCE_SUM_PRG=sha256sum SOURCE_FORMAT=tar.gz SOURCE_IN_SUBDIR=false diff --git a/conf/nginx.conf b/conf/nginx.conf new file mode 100644 index 0000000..6271ec8 --- /dev/null +++ b/conf/nginx.conf @@ -0,0 +1,15 @@ +location __PATH__ { + #Source: https://docs.weblate.org/en/latest/admin/install.html#sample-configuration-for-nginx-and-uwsgi + # Path to source + alias __FINALPATH__/ ; + + include uwsgi_params; + # Needed for long running operations in admin interface + uwsgi_read_timeout 3600; + uwsgi_param SCRIPT_NAME __PATH__; + uwsgi_modifier1 30; + uwsgi_pass unix:///var/run/uwsgi/__NAME__.socket; + + # Include SSOWAT user panel. + include conf.d/yunohost_panel.conf.inc; +} diff --git a/conf/pagure.cfg.sample b/conf/pagure.cfg.sample new file mode 100644 index 0000000..158d650 --- /dev/null +++ b/conf/pagure.cfg.sample @@ -0,0 +1,210 @@ +import os +from datetime import timedelta + +### Set the time after which the admin session expires +# There are two sessions on pagure, login that holds for 31 days and +# the session defined here after which an user has to re-login. +# This session is used when accessing all administrative parts of pagure +# (ie: changing a project's or a user's settings) +ADMIN_SESSION_LIFETIME = timedelta(minutes=20) + +### Secret key for the Flask application +SECRET_KEY='__SECRET_KEY__' + +### url to the database server: +#DB_URL=mysql://user:pass@host/db_name +DB_URL = 'postgres://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__' +#DB_URL = 'sqlite:////var/tmp/pagure_dev.sqlite' + +### The FAS group in which the admin of pagure are +ADMIN_GROUP = ['sysadmin-main'] + +### Hard-coded list of global admins +PAGURE_ADMIN_USERS = [] + +### The email address to which the flask.log will send the errors (tracebacks) +EMAIL_ERROR = 'root@__DOMAIN__' + +### SMTP settings +SMTP_SERVER = 'localhost' +SMTP_PORT = 587 +SMTP_SSL = True + +#Specify both for enabling SMTP with auth +SMTP_USERNAME = None +SMTP_PASSWORD = None + +### Information used to sent notifications +FROM_EMAIL = 'pagure@__DOMAIN__' +DOMAIN_EMAIL_NOTIFICATIONS = '__DOMAIN__' +SALT_EMAIL = '__SALT_EMAIL__' + +### The URL at which the project is available. +APP_URL = 'https://__DOMAIN__/' +### The URL at which the documentation of projects will be available +## This should be in a different domain to avoid XSS issues since we want +## to allow raw html to be displayed (different domain, ie not a sub-domain). +DOC_APP_URL = None + +### The URL to use to clone git repositories. +GIT_URL_SSH = 'ssh://git@__DOMAIN__/' +GIT_URL_GIT = 'git://__DOMAIN__/' + +### Folder containing to the git repos +GIT_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'repos' +) + +### Folder containing the docs repos +DOCS_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'docs' +) + +### Folder containing the tickets repos +TICKETS_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'tickets' +) + +### Folder containing the pull-requests repos +REQUESTS_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'requests' +) + +### Folder containing the clones for the remote pull-requests +REMOTE_GIT_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'remotes' +) + +### Whether to enable scanning for viruses in attachments +VIRUS_SCAN_ATTACHMENTS = False + + +### Configuration file for gitolite +GITOLITE_CONFIG = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'gitolite.conf' +) + + +### Home folder of the gitolite user +### Folder where to run gl-compile-conf from +GITOLITE_HOME = None + +### Version of gitolite used: 2 or 3? +GITOLITE_VERSION = 3 + +### Folder containing all the public ssh keys for gitolite +GITOLITE_KEYDIR = None + +### Path to the gitolite.rc file +GL_RC = None + +### Path to the /bin directory where the gitolite tools can be found +GL_BINDIR = None + + +# SSH Information + +### The ssh certificates of the git server to be provided to the user +### /!\ format is important +# SSH_KEYS = {'RSA': {'fingerprint': '', 'pubkey': ''}} + + + +# Optional configuration + +### Number of items displayed per page +# Used when listing items +ITEM_PER_PAGE = 50 + +### Maximum size of the uploaded content +# Used to limit the size of file attached to a ticket for example +MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes + +### Lenght for short commits ids or file hex +SHORT_LENGTH = 6 + +### List of blacklisted project names that can conflicts for pagure's URLs +### or other +BLACKLISTED_PROJECTS = [ + 'static', 'pv', 'releases', 'new', 'api', 'settings', + 'logout', 'login', 'users', 'groups', 'projects'] + +### IP addresses allowed to access the internal endpoints +### These endpoints are used by the milter and are security sensitive, thus +### the IP filter +IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] + +### EventSource/Web-Hook/Redis configuration +# The eventsource integration is what allows pagure to refresh the content +# on your page when someone else comments on the ticket (and this without +# asking you to reload the page. +# By default it is off, ie: EVENTSOURCE_SOURCE is None, to turn it on, specify +# here what the URL of the eventsource server is, for example: +# https://ev.pagure.io or https://pagure.io:8080 or whatever you are using +# (Note: the urls sent to it start with a '/' so no need to add one yourself) +EVENTSOURCE_SOURCE = None +# Port where the event source server is running (maybe be the same port +# as the one specified in EVENTSOURCE_SOURCE or a different one if you +# have something running in front of the server such as apache or stunnel). +EVENTSOURCE_PORT = 8080 +# If this port is specified, the event source server will run another server +# at this port and will provide information about the number of active +# connections running on the first (main) event source server +#EV_STATS_PORT = 8888 +# Web-hook can be turned on or off allowing using them for notifications, or +# not. +WEBHOOK = False + +### Redis configuration +# A redis server is required for both the Event-Source server or the web-hook +# server. +REDIS_HOST = '0.0.0.0' +REDIS_PORT = 6379 +REDIS_DB = 0 + +# Authentication related configuration option + +### Switch the authentication method +# Specify which authentication method to use, defaults to `fas` can be or +# `local` +# Default: ``fas``. +PAGURE_AUTH = 'local' + +# When this is set to True, the session cookie will only be returned to the +# server via ssl (https). If you connect to the server via plain http, the +# cookie will not be sent. This prevents sniffing of the cookie contents. +# This may be set to False when testing your application but should always +# be set to True in production. +# Default: ``True``. +SESSION_COOKIE_SECURE = True + +# The name of the cookie used to store the session id. +# Default: ``.pagure``. +SESSION_COOKIE_NAME = 'pagure' + +# Boolean specifying whether to check the user's IP address when retrieving +# its session. This make things more secure (thus is on by default) but +# under certain setup it might not work (for example is there are proxies +# in front of the application). +CHECK_SESSION_IP = True + +# Used by SESSION_COOKIE_PATH +APPLICATION_ROOT = '/' + +# Allow the backward compatiblity endpoints for the old URLs schema to +# see the commits of a repo. This is only interesting if you pagure instance +# was running since before version 1.3 and if you care about backward +# compatibility in your URLs. +OLD_VIEW_COMMIT_ENABLED = False diff --git a/conf/pagure.wsgi b/conf/pagure.wsgi new file mode 100644 index 0000000..b945279 --- /dev/null +++ b/conf/pagure.wsgi @@ -0,0 +1,28 @@ +#-*- coding: utf-8 -*- + +# The three lines below are required to run on EL6 as EL6 has +# two possible version of python-sqlalchemy and python-jinja2 +# These lines make sure the application uses the correct version. +import __main__ +__main__.__requires__ = ['SQLAlchemy >= 0.8', 'jinja2 >= 2.4'] +import pkg_resources + +# Set the environment variable pointing to the configuration file +import os +os.environ['PAGURE_CONFIG'] = '__FINALPATH__/pagure.cfg' + +# Set the environment variable if the tmp folder needs to be moved +# Might be necessary to work around bug in libgit2: +# refs: https://github.com/libgit2/libgit2/issues/2965 +# and https://github.com/libgit2/libgit2/issues/2797 +os.environ['TEMP'] = '/var/tmp/' + +# The following is only needed if you did not install pagure +# as a python module (for example if you run it from a git clone). +import sys +sys.path.insert(0, '__FINALPATH__/pagure/') + + +# The most import line to make the wsgi working +from pagure import APP as application +#application.debug = True diff --git a/conf/systemd.service b/conf/systemd.service deleted file mode 100644 index 1d34019..0000000 --- a/conf/systemd.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Pagure server service -After=network.target - -[Service] -Type=simple -User=__APP__ -Group=__APP__ -WorkingDirectory=__FINALPATH__ -ExecStart=__FINALPATH__/pagure/runworker.py --debug & __FINALPATH__/pagure/runserver.py --debug - -[Install] -WantedBy=multi-user.target diff --git a/conf/uwsgi-app@.service b/conf/uwsgi-app@.service new file mode 100644 index 0000000..c4603d4 --- /dev/null +++ b/conf/uwsgi-app@.service @@ -0,0 +1,16 @@ +[Unit] +Description=%i uWSGI app +After=syslog.target + +[Service] +ExecStart=/usr/bin/uwsgi \ + --ini /etc/uwsgi/apps-available/%i.ini \ + --socket /var/run/uwsgi/%i.socket \ + --logto /var/log/uwsgi/app/%i +User=%i +Group=www-data +Restart=on-failure +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all diff --git a/conf/uwsgi-app@.socket b/conf/uwsgi-app@.socket new file mode 100644 index 0000000..0975e29 --- /dev/null +++ b/conf/uwsgi-app@.socket @@ -0,0 +1,11 @@ +[Unit] +Description=Socket for uWSGI app %i + +[Socket] +ListenStream=/var/run/uwsgi/%i.socket +SocketUser=%i +SocketGroup=www-data +SocketMode=0775 + +[Install] +WantedBy=sockets.target diff --git a/conf/uwsgi.ini b/conf/uwsgi.ini new file mode 100644 index 0000000..9b1ef87 --- /dev/null +++ b/conf/uwsgi.ini @@ -0,0 +1,12 @@ +[uwsgi] +plugins = python +master = true +protocol = uwsgi +socket = /var/run/uwsgi/__APP__.socket +virtualenv = __FINALPATH__/venv +wsgi-file = __FINALPATH__/pagure.wsgi +## master = [master process (true of false)] +master = true +## processes = [number of processes] +processes = 5 + diff --git a/manifest.json b/manifest.json index 01f16c4..b9270be 100644 --- a/manifest.json +++ b/manifest.json @@ -3,12 +3,12 @@ "id": "pagure", "packaging_format": 1, "requirements": { - "yunohost": ">= 2.7.0" + "yunohost": ">= 3.0.0" }, "description": { - "en": "A translation platform using Git and Python" + "en": "Pagure is a git-centered forge, python based using pygit2." }, - "version": "3.8.0", + "version": "3.10.1", "url": "https://pagure.io/pagure", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index f6dfbd9..157262c 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,5 +1,82 @@ #!/bin/bash +ynh_check_global_uwsgi_config () { + uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" + + if [ -f /etc/systemd/system/uwsgi-app@.service ]; + then + echo "Uwsgi generic file is already installed" + else + cp ../conf/uwsgi-app@.socket /etc/systemd/system/uwsgi-app@.socket + cp ../conf/uwsgi-app@.service /etc/systemd/system/uwsgi-app@.service + fi + + # make sure the folder for sockets exists and set authorizations + mkdir -p /var/run/uwsgi/ + chown root:www-data /var/run/uwsgi/ + chmod -R 775 /var/run/uwsgi/ + + # make sure the folder for logs exists and set authorizations + mkdir -p /var/log/uwsgi/app/ + chown root:www-data /var/log/uwsgi/app/ + chmod -R 775 /var/log/uwsgi/app/ +} + +# Create a dedicated uwsgi ini file to use with generic uwsgi service +# It will install generic uwsgi.socket and +# +# This will use a template in ../conf/uwsgi.ini +# and will replace the following keywords with +# global variables that should be defined before calling +# this helper : +# +# __APP__ by $app +# __FINALPATH__ by $final_path +# +# usage: ynh_add_systemd_config +ynh_add_uwsgi_service () { + ynh_check_global_uwsgi_config + + # www-data group is needed since it is this nginx who will start the service + usermod --append --groups www-data "$app" || ynh_die "It wasn't possible to add user $app to group www-data" + + finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" + ynh_backup_if_checksum_is_different "$finaluwsgiini" + cp ../conf/uwsgi.ini "$finaluwsgiini" + + # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. + # Substitute in a nginx config file only if the variable is not empty + if test -n "${final_path:-}"; then + ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini" + fi + if test -n "${app:-}"; then + ynh_replace_string "__APP__" "$app" "$finaluwsgiini" + fi + ynh_store_file_checksum "$finaluwsgiini" + + chown root: "$finaluwsgiini" + systemctl enable "uwsgi-app@$app.socket" + systemctl start "uwsgi-app@$app.socket" + systemctl daemon-reload + + # Add as a service + yunohost service add "uwsgi-app@$app.socket" --log "/var/log/uwsgi/app/$app" +} + +# Remove the dedicated uwsgi ini file +# +# usage: ynh_remove_systemd_config +ynh_remove_uwsgi_service () { + finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" + if [ -e "$finaluwsgiini" ]; then + systemctl stop "uwsgi-app@$app.socket" + systemctl disable "uwsgi-app@$app.socket" + yunohost service remove "uwsgi-app@$app.socket" + + ynh_secure_remove "$finaluwsgiini" + fi +} + ynh_psql_test_if_first_run() { if [ -f /etc/yunohost/psql ]; then diff --git a/scripts/install b/scripts/install index 753afef..8a5e219 100755 --- a/scripts/install +++ b/scripts/install @@ -73,7 +73,31 @@ ynh_app_setting_set "$app" final_path "$final_path" #================================================= ynh_install_app_dependencies git python-virtualenv libgit2-dev \ - libjpeg-dev gcc libffi-dev python-dev python-cffi + libjpeg-dev gcc libffi-dev python-dev python-cffi \ + postgresql python-psycopg2 \ + uwsgi uwsgi-plugin-python + +#================================================= +# CREATE A PostgreSQL DATABASE +#================================================= + +ynh_psql_test_if_first_run + +db_name=$(ynh_sanitize_dbid "$app") +db_pwd=$(ynh_string_random) +# Initialize database and store postgres password for upgrade +ynh_psql_create_db "$db_name" "$app" "$db_pwd" +ynh_app_setting_set "$app" db_name "$db_name" +ynh_app_setting_set "$app" db_pwd "$db_pwd" + +systemctl reload postgresql + +#================================================= +# NGINX CONFIGURATION +#================================================= + +# Create a dedicated nginx config +ynh_add_nginx_config #================================================= # CREATE DEDICATED USER @@ -84,13 +108,39 @@ ynh_system_user_create "$app" "${final_path}" #================================================= # SPECIFIC SETUP #================================================= +# setup pagure.cfg +#================================================= + +secret_key=$(ynh_string_random) +salt_email=$(ynh_string_random) + +cp ../conf/pagure.cfg.sample "$final_path/pagure.cfg" +ynh_replace_string "__SECRET_KEY__" "$secret_key" "$final_path/pagure.cfg" +ynh_replace_string "__DB_USER__" "$app" "$final_path/pagure.cfg" +ynh_replace_string "__DB_PWD__" "$db_pwd" "$final_path/pagure.cfg" +ynh_replace_string "__DB_NAME__" "$db_name" "$final_path/pagure.cfg" +ynh_replace_string "__DOMAIN__" "$domain" "$final_path/pagure.cfg" +ynh_replace_string "__SALT_EMAIL__" "$salt_email" "$final_path/pagure.cfg" + +#================================================= +# setup pagure.wsgi +#================================================= + +cp ../conf/pagure.wsgi "$final_path/pagure.wsgi" +ynh_replace_string "__FINALPATH__" "$final_path" "$final_path/pagure.wsgi" + +#================================================= +# setup uwsgi service +#================================================= + +ynh_add_uwsgi_service #================================================= # Get Pagure source #================================================= ynh_setup_source "${final_path}" -ln -s "${final_path}/pagure-3.8" "${final_path}/pagure" +ln -s "${final_path}/pagure-3.10.1" "${final_path}/pagure" #================================================= # PIP INSTALLATION @@ -101,13 +151,13 @@ virtualenv "${final_path}/venv" set +eu source "${final_path}/venv/bin/activate" "${final_path}/venv/bin/pip" install cffi - "${final_path}/venv/bin/pip" install pygit2==0.21 + "${final_path}/venv/bin/pip" install pygit2==0.24 "${final_path}/venv/bin/pip" install -r "${final_path}/pagure/requirements.txt" + "${final_path}/venv/bin/pip" install psycopg2 ) #================================================= # SPECIFIC SETUP Filling up the database -# https://docs.weblate.org/en/latest/admin/install.html#filling-up-the-database #========================================== # Create the folder that will receive the projects, forks, docs, requests and tickets' git repo: @@ -117,18 +167,9 @@ mkdir -p "${final_path}/lcl/{repos,docs,forks,tickets,requests,remotes,attachmen set +eu source "${final_path}/venv/bin/activate" cd "${final_path}/pagure" - python createdb.py + PAGURE_CONFIG=${final_path}/pagure.cfg python createdb.py ) -#================================================= -# Set-up a service -#================================================= -# TODO: add port support for service -# server_port=$(ynh_find_port 8080) -# ynh_app_setting_set "$app" port "$server_port" - -ynh_add_systemd_config - #================================================= # GENERIC FINALIZATION #================================================= @@ -138,13 +179,6 @@ ynh_add_systemd_config # Set permissions to app files chown -R "$app": "$final_path" -#================================================= -# SETUP LOGROTATE -#================================================= - -# Use logrotate to manage application logfile(s) -ynh_use_logrotate - #================================================= # SETUP SSOWAT #================================================= @@ -164,5 +198,4 @@ fi # RELOAD NGINX #================================================= -systemctl start "$app.service" systemctl reload nginx diff --git a/scripts/remove b/scripts/remove index 811e39f..d5f2c66 100755 --- a/scripts/remove +++ b/scripts/remove @@ -16,17 +16,20 @@ source /usr/share/yunohost/helpers app=$YNH_APP_INSTANCE_NAME db_name=$(ynh_app_setting_get "$app" db_name) +db_name=$(ynh_app_setting_get "$app" domain) #================================================= -# REMOVE SERVICE FROM ADMIN PANEL +# REMOVE uwsgi and systemd files #================================================= -if yunohost service status | grep -q "$app" -then - echo "Remove $app service" - systemctl stop "$app.service" - yunohost service remove "$app.service" -fi +ynh_remove_uwsgi_service + +#================================================= +# REMOVE THE PostgreSQL DATABASE +#================================================= + +# Remove a database if it exists, along with the associated user +ynh_psql_remove_db "$db_name" "$app" #================================================= # REMOVE DEPENDENCIES @@ -43,17 +46,17 @@ ynh_remove_app_dependencies ynh_secure_remove "/var/www/$app" #================================================= -# REMOVE LOGROTATE CONFIGURATION +# REMOVE NGINX CONFIGURATION #================================================= -# Remove the app-specific logrotate config -ynh_remove_logrotate +# Remove the dedicated nginx config +ynh_remove_nginx_config #================================================= -# SPECIFIC REMOVE +# REMOVE uwsgi and systemd files #================================================= -ynh_remove_systemd_config +ynh_remove_uwsgi_service #================================================= # GENERIC FINALIZATION