From 72dd852b5ef7a4b0de34839a33e5a24acb73180f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Mon, 10 Sep 2018 23:05:26 +0200 Subject: [PATCH 01/13] Set up to 4 uwsgi processes --- conf/uwsgi-app@.service | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/uwsgi-app@.service b/conf/uwsgi-app@.service index 888994d..b1c02b7 100644 --- a/conf/uwsgi-app@.service +++ b/conf/uwsgi-app@.service @@ -8,6 +8,7 @@ ExecStart=/usr/bin/uwsgi \ --socket /var/run/uwsgi/%i.socket \ --chmod-socket=775 \ --logto /var/log/uwsgi/app/%i + --processes 4 User=%i Group=www-data Restart=on-failure From 1fa33cd2326bdf46fafab6188feed8042b728e02 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Mon, 17 Sep 2018 22:29:37 +0200 Subject: [PATCH 02/13] Improve Readme --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 73b3ff3..cde1ee4 100644 --- a/README.md +++ b/README.md @@ -53,3 +53,27 @@ Carefull, this application install PostgreSQL and change the default configurati ## LDAP connexion It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/latest/admin/auth.html?highlight=LDAP#ldap-authentication), I'm unsure it is a good idea to connect this kind of tools to your LDAP. + +## State of this package + +* works fine: +[x] install/remove/backup/remove/upgrade with x86_64 + +* to be confirmed: +[ ] ARM support + +* to be added: +[ ] strenghen uwsgi systemd security (https://github.com/YunoHost-Apps/kresus_ynh/pull/21/commits/0ce4979d5037b111aa9698ed5552135bf0ed7165) +[ ] use jq instead of grep/sed +[ ] change URL script +[ ] use debian package for lxml (may unlock ARM support) +[ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445) +[ ] Add fail2ban script + +## Links + + * Report a bug about this package: https://github.com/YunoHost-Apps/weblate_ynh + * Report a bug about Weblate itself: https://github.com/WeblateOrg/weblate + * Documentation: https://docs.weblate.org/ + * Weblate website: https://weblate.org/ + * YunoHost website: https://yunohost.org/ From eb87fcf289ebe0fc0049d89111596126e31dab5b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Sat, 6 Oct 2018 20:49:54 +0200 Subject: [PATCH 03/13] typo fix --- conf/uwsgi-app@.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uwsgi-app@.service b/conf/uwsgi-app@.service index b1c02b7..8b316e2 100644 --- a/conf/uwsgi-app@.service +++ b/conf/uwsgi-app@.service @@ -7,7 +7,7 @@ ExecStart=/usr/bin/uwsgi \ --ini /etc/uwsgi/apps-available/%i.ini \ --socket /var/run/uwsgi/%i.socket \ --chmod-socket=775 \ - --logto /var/log/uwsgi/app/%i + --logto /var/log/uwsgi/app/%i \ --processes 4 User=%i Group=www-data From 6647a93af3ccdd9abdfddcb20b9260bfe36c3789 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Sat, 6 Oct 2018 20:50:23 +0200 Subject: [PATCH 04/13] Weblate 3.2 initialization --- conf/settings_history/settings.3.2.0.py | 770 ++++++++++++++++++++++++ manifest.json | 4 +- scripts/_common.sh | 2 +- 3 files changed, 773 insertions(+), 3 deletions(-) create mode 100644 conf/settings_history/settings.3.2.0.py diff --git a/conf/settings_history/settings.3.2.0.py b/conf/settings_history/settings.3.2.0.py new file mode 100644 index 0000000..37193f5 --- /dev/null +++ b/conf/settings_history/settings.3.2.0.py @@ -0,0 +1,770 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2012 - 2018 Michal Čihař +# +# This file is part of Weblate +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from __future__ import unicode_literals +import platform +import os +from logging.handlers import SysLogHandler + +# +# Django settings for Weblate project. +# + +DEBUG = False + +ADMINS = ( + ('__ADMIN__', '__ADMINMAIL__'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + # Database engine + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + # Database name + 'NAME': '__NAME__', + # Database user + 'USER': '__NAME__', + # Database password + 'PASSWORD': '__DB_PWD__', + # Set to empty string for localhost + 'HOST': 'localhost', + # Set to empty string for default + 'PORT': '5432', + } +} + +BASE_DIR = '__FINALPATH__' + +# Data directory +DATA_DIR = os.path.join(BASE_DIR, 'data') + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# In a Windows environment this must be set to your system time zone. +TIME_ZONE = 'UTC' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +LANGUAGES = ( + ('ar', 'العربية'), + ('az', 'Azərbaycan'), + ('be', 'Беларуская'), + ('be@latin', 'Biełaruskaja'), + ('bg', 'Български'), + ('br', 'Brezhoneg'), + ('ca', 'Català'), + ('cs', 'Čeština'), + ('da', 'Dansk'), + ('de', 'Deutsch'), + ('en', 'English'), + ('en-gb', 'English (United Kingdom)'), + ('el', 'Ελληνικά'), + ('es', 'Español'), + ('fi', 'Suomi'), + ('fr', 'Français'), + ('fy', 'Frysk'), + ('gl', 'Galego'), + ('he', 'עברית'), + ('hu', 'Magyar'), + ('id', 'Indonesia'), + ('it', 'Italiano'), + ('ja', '日本語'), + ('ko', '한국어'), + ('ksh', 'Kölsch'), + ('nb', 'Norsk bokmål'), + ('nl', 'Nederlands'), + ('pl', 'Polski'), + ('pt', 'Português'), + ('pt-br', 'Português brasileiro'), + ('ru', 'Русский'), + ('sk', 'Slovenčina'), + ('sl', 'Slovenščina'), + ('sr', 'Српски'), + ('sv', 'Svenska'), + ('tr', 'Türkçe'), + ('uk', 'Українська'), + ('zh-hans', '简体字'), + ('zh-hant', '正體字'), +) + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale. +USE_L10N = True + +# If you set this to False, Django will not use timezone-aware datetimes. +USE_TZ = True + +# URL prefix to use, please see documentation for more details +URL_PREFIX = '__PATHURL__' + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_ROOT = os.path.join(DATA_DIR, 'media') + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '{0}/media/'.format(URL_PREFIX) + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = os.path.join(DATA_DIR, 'static') + +# URL prefix for static files. +# Example: "http://media.lawrence.com/static/" +STATIC_URL = '{0}/static/'.format(URL_PREFIX) + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'compressor.finders.CompressorFinder', +) + +# Make this unique, and don't share it with anybody. +# You can generate it using examples/generate-secret-key +SECRET_KEY = '__KEY__' # noqa + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + os.path.join(BASE_DIR, 'weblate', 'templates'), + ], + 'OPTIONS': { + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.request', + 'django.template.context_processors.csrf', + 'django.contrib.messages.context_processors.messages', + 'weblate.trans.context_processors.weblate_context', + ], + 'loaders': [ + ('django.template.loaders.cached.Loader', [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ]), + ], + }, + }, +] + + +# GitHub username for sending pull requests. +# Please see the documentation for more details. +GITHUB_USERNAME = "__GITHUBUSER__" + +# Authentication configuration +AUTHENTICATION_BACKENDS = ( + 'social_core.backends.email.EmailAuth', + # 'social_core.backends.google.GoogleOAuth2', + # 'social_core.backends.github.GithubOAuth2', + # 'social_core.backends.bitbucket.BitbucketOAuth', + 'social_core.backends.suse.OpenSUSEOpenId', + 'social_core.backends.ubuntu.UbuntuOpenId', + 'social_core.backends.fedora.FedoraOpenId', + # 'social_core.backends.facebook.FacebookOAuth2', + 'weblate.accounts.auth.WeblateUserBackend', +) + +# Custom user model +AUTH_USER_MODEL = 'weblate_auth.User' + +# Social auth backends setup +SOCIAL_AUTH_GITHUB_KEY = '' +SOCIAL_AUTH_GITHUB_SECRET = '' +SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] + +SOCIAL_AUTH_BITBUCKET_KEY = '' +SOCIAL_AUTH_BITBUCKET_SECRET = '' +SOCIAL_AUTH_BITBUCKET_VERIFIED_EMAILS_ONLY = True + +SOCIAL_AUTH_FACEBOOK_KEY = '' +SOCIAL_AUTH_FACEBOOK_SECRET = '' +SOCIAL_AUTH_FACEBOOK_SCOPE = ['email', 'public_profile'] + +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' + +# Social auth settings +SOCIAL_AUTH_PIPELINE = ( + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'weblate.accounts.pipeline.store_params', + 'weblate.accounts.pipeline.verify_open', + 'social_core.pipeline.user.get_username', + 'weblate.accounts.pipeline.require_email', + 'social_core.pipeline.mail.mail_validation', + 'weblate.accounts.pipeline.revoke_mail_code', + 'weblate.accounts.pipeline.ensure_valid', + 'weblate.accounts.pipeline.remove_account', + 'social_core.pipeline.social_auth.associate_by_email', + 'weblate.accounts.pipeline.reauthenticate', + 'weblate.accounts.pipeline.verify_username', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'weblate.accounts.pipeline.cleanup_next', + 'weblate.accounts.pipeline.user_full_name', + 'weblate.accounts.pipeline.store_email', + 'weblate.accounts.pipeline.notify_connect', + 'weblate.accounts.pipeline.password_reset', +) +SOCIAL_AUTH_DISCONNECT_PIPELINE = ( + 'social_core.pipeline.disconnect.allowed_to_disconnect', + 'social_core.pipeline.disconnect.get_entries', + 'social_core.pipeline.disconnect.revoke_tokens', + 'weblate.accounts.pipeline.cycle_session', + 'weblate.accounts.pipeline.adjust_primary_mail', + 'weblate.accounts.pipeline.notify_disconnect', + 'social_core.pipeline.disconnect.disconnect', + 'weblate.accounts.pipeline.cleanup_next', +) + +# Custom authentication strategy +SOCIAL_AUTH_STRATEGY = 'weblate.accounts.strategy.WeblateStrategy' + +# Raise exceptions so that we can handle them later +SOCIAL_AUTH_RAISE_EXCEPTIONS = True + +SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = \ + 'weblate.accounts.pipeline.send_validation' +SOCIAL_AUTH_EMAIL_VALIDATION_URL = \ + '{0}/accounts/email-sent/'.format(URL_PREFIX) +SOCIAL_AUTH_LOGIN_ERROR_URL = \ + '{0}/accounts/login/'.format(URL_PREFIX) +SOCIAL_AUTH_EMAIL_FORM_URL = \ + '{0}/accounts/email/'.format(URL_PREFIX) +SOCIAL_AUTH_NEW_ASSOCIATION_REDIRECT_URL = \ + '{0}/accounts/profile/#auth'.format(URL_PREFIX) +SOCIAL_AUTH_PROTECTED_USER_FIELDS = ('email',) +SOCIAL_AUTH_SLUGIFY_USERNAMES = True +SOCIAL_AUTH_SLUGIFY_FUNCTION = 'weblate.accounts.pipeline.slugify_username' + +# Password validation configuration +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + 'OPTIONS': { + 'min_length': 6, + } + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, + { + 'NAME': 'weblate.accounts.password_validation.CharsPasswordValidator', + }, + { + 'NAME': 'weblate.accounts.password_validation.PastPasswordsValidator', + }, + # Optional password strength validation by django-zxcvbn-password + # { + # 'NAME': 'zxcvbn_password.ZXCVBNValidator', + # 'OPTIONS': { + # 'min_score': 3, + # 'user_attributes': ('username', 'email', 'full_name') + # } + # }, +] + +# Middleware +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'weblate.accounts.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'social_django.middleware.SocialAuthExceptionMiddleware', + 'weblate.accounts.middleware.RequireLoginMiddleware', + 'weblate.middleware.SecurityMiddleware', + 'weblate.wladmin.middleware.ConfigurationErrorsMiddleware', +] + +ROOT_URLCONF = 'weblate.urls' + +# Django and Weblate apps +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin.apps.SimpleAdminConfig', + 'django.contrib.admindocs', + 'django.contrib.sitemaps', + 'social_django', + 'crispy_forms', + 'compressor', + 'rest_framework', + 'rest_framework.authtoken', + 'weblate.addons', + 'weblate.auth', + 'weblate.checks', + 'weblate.formats', + 'weblate.machinery', + 'weblate.trans', + 'weblate.lang', + 'weblate.langdata', + 'weblate.memory', + 'weblate.screenshots', + 'weblate.accounts', + 'weblate.utils', + 'weblate.vcs', + 'weblate.wladmin', + 'weblate', + + # Optional: Git exporter + # 'weblate.gitexport', +) + +# Path to locales +LOCALE_PATHS = (os.path.join(BASE_DIR, 'weblate', 'locale'), ) + +# Custom exception reporter to include some details +DEFAULT_EXCEPTION_REPORTER_FILTER = \ + 'weblate.trans.debug.WeblateExceptionReporterFilter' + +# Default logging of Weblate messages +# - to syslog in production (if available) +# - otherwise to console +# - you can also choose 'logfile' to log into separate file +# after configuring it below + +# Detect if we can connect to syslog +HAVE_SYSLOG = False +if platform.system() != 'Windows': + try: + handler = SysLogHandler( + address='/dev/log', facility=SysLogHandler.LOG_LOCAL2 + ) + handler.close() + HAVE_SYSLOG = True + except IOError: + HAVE_SYSLOG = False + +if DEBUG or not HAVE_SYSLOG: + DEFAULT_LOG = 'console' +else: + DEFAULT_LOG = 'syslog' + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/stable/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'syslog': { + 'format': 'weblate[%(process)d]: %(levelname)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + 'logfile': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'django.server': { + '()': 'django.utils.log.ServerFormatter', + 'format': '[%(server_time)s] %(message)s', + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler', + 'include_html': True, + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'django.server': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'django.server', + }, + 'syslog': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'formatter': 'syslog', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL2, + }, + # Logging to a file + # 'logfile': { + # 'level':'DEBUG', + # 'class':'logging.handlers.RotatingFileHandler', + # 'filename': "/var/log/weblate/weblate.log", + # 'maxBytes': 100000, + # 'backupCount': 3, + # 'formatter': 'logfile', + # }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', DEFAULT_LOG], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.server': { + 'handlers': ['django.server'], + 'level': 'INFO', + 'propagate': False, + }, + # Logging database queries + # 'django.db.backends': { + # 'handlers': [DEFAULT_LOG], + # 'level': 'DEBUG', + # }, + 'weblate': { + 'handlers': [DEFAULT_LOG], + 'level': 'DEBUG', + }, + # Logging VCS operations + # 'weblate-vcs': { + # 'handlers': [DEFAULT_LOG], + # 'level': 'DEBUG', + # }, + # Python Social Auth logging + # 'social': { + # 'handlers': [DEFAULT_LOG], + # 'level': 'DEBUG', + # }, + } +} + +# Logging of management commands to console +if (os.environ.get('DJANGO_IS_MANAGEMENT_COMMAND', False) and + 'console' not in LOGGING['loggers']['weblate']['handlers']): + LOGGING['loggers']['weblate']['handlers'].append('console') + +# Remove syslog setup if it's not present +if not HAVE_SYSLOG: + del LOGGING['handlers']['syslog'] + +# List of machine translations +# MT_SERVICES = ( +# 'weblate.machinery.apertium.ApertiumAPYTranslation', +# 'weblate.machinery.deepl.DeepLTranslation', +# 'weblate.machinery.glosbe.GlosbeTranslation', +# 'weblate.machinery.google.GoogleTranslation', +# 'weblate.machinery.microsoft.MicrosoftCognitiveTranslation', +# 'weblate.machinery.mymemory.MyMemoryTranslation', +# 'weblate.machinery.tmserver.AmagamaTranslation', +# 'weblate.machinery.tmserver.TMServerTranslation', +# 'weblate.machinery.yandex.YandexTranslation', +# 'weblate.machinery.weblatetm.WeblateTranslation', +# 'weblate.machinery.saptranslationhub.SAPTranslationHub', +# 'weblate.memory.machine.WeblateMemory', +# ) + +# Machine translation API keys + +# URL of the Apertium APy server +MT_APERTIUM_APY = None + +# DeepL API key +MT_DEEPL_KEY = None + +# Microsoft Cognitive Services Translator API, register at +# https://portal.azure.com/ +MT_MICROSOFT_COGNITIVE_KEY = None + +# MyMemory identification email, see +# https://mymemory.translated.net/doc/spec.php +MT_MYMEMORY_EMAIL = None + +# Optional MyMemory credentials to access private translation memory +MT_MYMEMORY_USER = None +MT_MYMEMORY_KEY = None + +# Google API key for Google Translate API +MT_GOOGLE_KEY = None + +# API key for Yandex Translate API +MT_YANDEX_KEY = None + +# tmserver URL +MT_TMSERVER = None + +# SAP Translation Hub +MT_SAP_BASE_URL = None +MT_SAP_SANDBOX_APIKEY = None +MT_SAP_USERNAME = None +MT_SAP_PASSWORD = None +MT_SAP_USE_MT = True + +# Title of site to use +SITE_TITLE = 'Weblate' + +# Whether site uses https +ENABLE_HTTPS = True + +# Use HTTPS when creating redirect URLs for social authentication, see +# documentation for more details: +# http://python-social-auth-docs.readthedocs.io/en/latest/configuration/settings.html#processing-redirects-and-urlopen +SOCIAL_AUTH_REDIRECT_IS_HTTPS = ENABLE_HTTPS + +# Make CSRF cookie HttpOnly, see documentation for more details: +# https://docs.djangoproject.com/en/1.11/ref/settings/#csrf-cookie-httponly +CSRF_COOKIE_HTTPONLY = True +CSRF_COOKIE_SECURE = ENABLE_HTTPS +# Store CSRF token in session (since Django 1.11) +CSRF_USE_SESSIONS = True +SESSION_COOKIE_SECURE = ENABLE_HTTPS +# SSL redirect +SECURE_SSL_REDIRECT = ENABLE_HTTPS +# Session cookie age (in seconds) +SESSION_COOKIE_AGE = 1209600 + +# Some security headers +SECURE_BROWSER_XSS_FILTER = True +X_FRAME_OPTIONS = 'DENY' +SECURE_CONTENT_TYPE_NOSNIFF = True + +# Optionally enable HSTS +SECURE_HSTS_SECONDS = 0 +SECURE_HSTS_PRELOAD = False +SECURE_HSTS_INCLUDE_SUBDOMAINS = False + +# URL of login +LOGIN_URL = '{0}/accounts/login/'.format(URL_PREFIX) + +# URL of logout +LOGOUT_URL = '{0}/accounts/logout/'.format(URL_PREFIX) + +# Default location for login +LOGIN_REDIRECT_URL = '{0}/'.format(URL_PREFIX) + +# Anonymous user name +ANONYMOUS_USER_NAME = 'anonymous' + +# Reverse proxy settings +IP_BEHIND_REVERSE_PROXY = False +IP_PROXY_HEADER = 'HTTP_X_FORWARDED_FOR' +IP_PROXY_OFFSET = 0 + +# Sending HTML in mails +EMAIL_SEND_HTML = True + +# Subject of emails includes site title +EMAIL_SUBJECT_PREFIX = '[{0}] '.format(SITE_TITLE) + +EMAIL_BACKEND = 'django_sendmail_backend.backends.EmailBackend' + +# Enable remote hooks +ENABLE_HOOKS = True + +# Whether to run hooks in background +BACKGROUND_HOOKS = True + +# Number of nearby messages to show in each direction +NEARBY_MESSAGES = 5 + +# Offload indexing +OFFLOAD_INDEXING = True + +# Use simple language codes for default language/country combinations +SIMPLIFY_LANGUAGES = True + +# Render forms using bootstrap +CRISPY_TEMPLATE_PACK = 'bootstrap3' + +# List of quality checks +# CHECK_LIST = ( +# 'weblate.checks.same.SameCheck', +# 'weblate.checks.chars.BeginNewlineCheck', +# 'weblate.checks.chars.EndNewlineCheck', +# 'weblate.checks.chars.BeginSpaceCheck', +# 'weblate.checks.chars.EndSpaceCheck', +# 'weblate.checks.chars.EndStopCheck', +# 'weblate.checks.chars.EndColonCheck', +# 'weblate.checks.chars.EndQuestionCheck', +# 'weblate.checks.chars.EndExclamationCheck', +# 'weblate.checks.chars.EndEllipsisCheck', +# 'weblate.checks.chars.EndSemicolonCheck', +# 'weblate.checks.chars.MaxLengthCheck', +# 'weblate.checks.format.PythonFormatCheck', +# 'weblate.checks.format.PythonBraceFormatCheck', +# 'weblate.checks.format.PHPFormatCheck', +# 'weblate.checks.format.CFormatCheck', +# 'weblate.checks.format.PerlFormatCheck', +# 'weblate.checks.format.JavascriptFormatCheck', +# 'weblate.checks.consistency.PluralsCheck', +# 'weblate.checks.consistency.SamePluralsCheck', +# 'weblate.checks.consistency.ConsistencyCheck', +# 'weblate.checks.consistency.TranslatedCheck', +# 'weblate.checks.chars.NewlineCountingCheck', +# 'weblate.checks.markup.BBCodeCheck', +# 'weblate.checks.chars.ZeroWidthSpaceCheck', +# 'weblate.checks.markup.XMLValidityCheck', +# 'weblate.checks.markup.XMLTagsCheck', +# 'weblate.checks.source.OptionalPluralCheck', +# 'weblate.checks.source.EllipsisCheck', +# 'weblate.checks.source.MultipleFailingCheck', +# ) + +# List of automatic fixups +# AUTOFIX_LIST = ( +# 'weblate.trans.autofixes.whitespace.SameBookendingWhitespace', +# 'weblate.trans.autofixes.chars.ReplaceTrailingDotsWithEllipsis', +# 'weblate.trans.autofixes.chars.RemoveZeroSpace', +# 'weblate.trans.autofixes.chars.RemoveControlChars', +# ) + +# List of enabled addons +# WEBLATE_ADDONS = ( +# 'weblate.addons.gettext.GenerateMoAddon', +# 'weblate.addons.gettext.UpdateLinguasAddon', +# 'weblate.addons.gettext.UpdateConfigureAddon', +# 'weblate.addons.gettext.MsgmergeAddon', +# 'weblate.addons.gettext.GettextCustomizeAddon', +# 'weblate.addons.gettext.GettextAuthorComments', +# 'weblate.addons.cleanup.CleanupAddon', +# 'weblate.addons.consistency.LangaugeConsistencyAddon', +# 'weblate.addons.discovery.DiscoveryAddon', +# 'weblate.addons.flags.SourceEditAddon', +# 'weblate.addons.flags.TargetEditAddon', +# 'weblate.addons.generate.GenerateFileAddon', +# 'weblate.addons.json.JSONCustomizeAddon', +# 'weblate.addons.properties.PropertiesSortAddon', +# ) + +# E-mail address that error messages come from. +SERVER_EMAIL = 'noreply@__DOMAIN__' + +# Default email address to use for various automated correspondence from +# the site managers. Used for registration emails. +DEFAULT_FROM_EMAIL = '__ADMINMAIL__' + +# List of URLs your site is supposed to serve +ALLOWED_HOSTS = ['__DOMAIN__'] + +# Example configuration to use memcached for caching +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': '127.0.0.1:__MEMCPORT__', + }, + 'avatar': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': os.path.join(BASE_DIR, 'avatar-cache'), + 'TIMEOUT': 3600, + 'OPTIONS': { + 'MAX_ENTRIES': 1000, + }, + } +} + +# REST framework settings for API +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + 'DEFAULT_PERMISSION_CLASSES': [ + 'rest_framework.permissions.IsAuthenticatedOrReadOnly' + ], + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework.authentication.TokenAuthentication', + 'weblate.api.authentication.BearerAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ), + 'DEFAULT_THROTTLE_CLASSES': ( + 'rest_framework.throttling.AnonRateThrottle', + 'rest_framework.throttling.UserRateThrottle' + ), + 'DEFAULT_THROTTLE_RATES': { + 'anon': '100/day', + 'user': '1000/day' + }, + 'DEFAULT_PAGINATION_CLASS': ( + 'rest_framework.pagination.PageNumberPagination' + ), + 'PAGE_SIZE': 20, + 'VIEW_DESCRIPTION_FUNCTION': 'weblate.api.views.get_view_description', + 'UNAUTHENTICATED_USER': 'weblate.auth.models.get_anonymous', +} + +# Example for restricting access to logged in users +# LOGIN_REQUIRED_URLS = ( +# r'/(.*)$', +# ) + +# In such case you will want to include some of the exceptions +# LOGIN_REQUIRED_URLS_EXCEPTIONS = ( +# r'/accounts/(.*)$', # Required for login +# r'/admin/login/(.*)$', # Required for admin login +# r'/static/(.*)$', # Required for development mode +# r'/widgets/(.*)$', # Allowing public access to widgets +# r'/data/(.*)$', # Allowing public access to data exports +# r'/hooks/(.*)$', # Allowing public access to notification hooks +# r'/healthz/$', # Allowing public access to health check +# r'/api/(.*)$', # Allowing access to API +# r'/js/i18n/$', # Javascript localization +# r'/contact/$', # Optional for contact form +# r'/legal/(.*)$', # Optional for legal app +# ) + +# Force sane test runner +TEST_RUNNER = 'django.test.runner.DiscoverRunner' diff --git a/manifest.json b/manifest.json index 1dfb4d8..38e70e0 100644 --- a/manifest.json +++ b/manifest.json @@ -3,12 +3,12 @@ "id": "weblate", "packaging_format": 1, "requirements": { - "yunohost": ">= 3.1.0" + "yunohost": ">= 3.2.0" }, "description": { "en": "A translation platform using Git and Python" }, - "version": "3.1.1~ynh2", + "version": "3.2.0~ynh1", "url": "https://weblate.org", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 50681bf..d412bf9 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,6 +1,6 @@ #!/bin/bash -current_version="3.1.1" +current_version="3.2.0" ynh_check_global_uwsgi_config () { uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" From 131f74288645b5e11e4cbac8f9d0dd5e6437601c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Sat, 6 Oct 2018 22:28:53 +0200 Subject: [PATCH 05/13] update settings for 3.2 --- README.md | 4 ++ conf/settings_history/settings.3.2.0.py | 69 +++++++++++++++++++++---- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cde1ee4..b4ba459 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/lat [ ] use debian package for lxml (may unlock ARM support) [ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445) [ ] Add fail2ban script +[ ] Use redis+hiredis for cache? +[ ] Enable CHECK_LIST? +[ ] Enable AUTOFIX_LIST? +[ ] Enable Translation Memory? ## Links diff --git a/conf/settings_history/settings.3.2.0.py b/conf/settings_history/settings.3.2.0.py index 37193f5..8800bc5 100644 --- a/conf/settings_history/settings.3.2.0.py +++ b/conf/settings_history/settings.3.2.0.py @@ -222,6 +222,8 @@ SOCIAL_AUTH_BITBUCKET_VERIFIED_EMAILS_ONLY = True SOCIAL_AUTH_FACEBOOK_KEY = '' SOCIAL_AUTH_FACEBOOK_SECRET = '' SOCIAL_AUTH_FACEBOOK_SCOPE = ['email', 'public_profile'] +SOCIAL_AUTH_FACEBOOK_PROFILE_EXTRA_PARAMS = {'fields': 'id,name,email'} +SOCIAL_AUTH_FACEBOOK_API_VERSION = '3.1' SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' @@ -329,7 +331,6 @@ MIDDLEWARE = [ 'social_django.middleware.SocialAuthExceptionMiddleware', 'weblate.accounts.middleware.RequireLoginMiddleware', 'weblate.middleware.SecurityMiddleware', - 'weblate.wladmin.middleware.ConfigurationErrorsMiddleware', ] ROOT_URLCONF = 'weblate.urls' @@ -507,16 +508,19 @@ if not HAVE_SYSLOG: # List of machine translations # MT_SERVICES = ( # 'weblate.machinery.apertium.ApertiumAPYTranslation', +# 'weblate.machinery.baidu.BaiduTranslation', # 'weblate.machinery.deepl.DeepLTranslation', # 'weblate.machinery.glosbe.GlosbeTranslation', # 'weblate.machinery.google.GoogleTranslation', # 'weblate.machinery.microsoft.MicrosoftCognitiveTranslation', +# 'weblate.machinery.microsoftterminology.MicrosoftTerminologyService', # 'weblate.machinery.mymemory.MyMemoryTranslation', # 'weblate.machinery.tmserver.AmagamaTranslation', # 'weblate.machinery.tmserver.TMServerTranslation', # 'weblate.machinery.yandex.YandexTranslation', # 'weblate.machinery.weblatetm.WeblateTranslation', # 'weblate.machinery.saptranslationhub.SAPTranslationHub', +# 'weblate.machinery.youdao.YoudaoTranslation', # 'weblate.memory.machine.WeblateMemory', # ) @@ -543,6 +547,14 @@ MT_MYMEMORY_KEY = None # Google API key for Google Translate API MT_GOOGLE_KEY = None +# Baidu app key and secret +MT_BAIDU_ID = None +MT_BAIDU_SECRET = None + +# Youdao Zhiyun app key and secret +MT_YOUDAO_ID = None +MT_YOUDAO_SECRET = None + # API key for Yandex Translate API MT_YANDEX_KEY = None @@ -617,15 +629,9 @@ EMAIL_BACKEND = 'django_sendmail_backend.backends.EmailBackend' # Enable remote hooks ENABLE_HOOKS = True -# Whether to run hooks in background -BACKGROUND_HOOKS = True - # Number of nearby messages to show in each direction NEARBY_MESSAGES = 5 -# Offload indexing -OFFLOAD_INDEXING = True - # Use simple language codes for default language/country combinations SIMPLIFY_LANGUAGES = True @@ -652,6 +658,10 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3' # 'weblate.checks.format.CFormatCheck', # 'weblate.checks.format.PerlFormatCheck', # 'weblate.checks.format.JavascriptFormatCheck', +# 'weblate.checks.format.CSharpFormatCheck', +# 'weblate.checks.format.JavaFormatCheck', +# 'weblate.checks.format.JavaMessageFormatCheck', +# 'weblate.checks.angularjs.AngularJSInterpolationCheck', # 'weblate.checks.consistency.PluralsCheck', # 'weblate.checks.consistency.SamePluralsCheck', # 'weblate.checks.consistency.ConsistencyCheck', @@ -702,7 +712,6 @@ DEFAULT_FROM_EMAIL = '__ADMINMAIL__' # List of URLs your site is supposed to serve ALLOWED_HOSTS = ['__DOMAIN__'] -# Example configuration to use memcached for caching CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', @@ -718,6 +727,35 @@ CACHES = { } } +# Example configuration for caching +# CACHES = { +# Recommended redis + hiredis: +# 'default': { +# 'BACKEND': 'django_redis.cache.RedisCache', +# 'LOCATION': 'redis://127.0.0.1:6379/0', +# # If redis is running on same host as Weblate, you might +# # want to use unix sockets instead: +# # 'LOCATION': 'unix:///var/run/redis/redis.sock?db=0', +# 'OPTIONS': { +# 'CLIENT_CLASS': 'django_redis.client.DefaultClient', +# 'PARSER_CLASS': 'redis.connection.HiredisParser', +# } +# }, +# Memcached alternative: +# 'default': { +# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION': '127.0.0.1:11211', +# }, +# 'avatar': { +# 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', +# 'LOCATION': os.path.join(DATA_DIR, 'avatar-cache'), +# 'TIMEOUT': 3600, +# 'OPTIONS': { +# 'MAX_ENTRIES': 1000, +# }, +# } +# } + # REST framework settings for API REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, @@ -766,5 +804,16 @@ REST_FRAMEWORK = { # r'/legal/(.*)$', # Optional for legal app # ) -# Force sane test runner -TEST_RUNNER = 'django.test.runner.DiscoverRunner' +# Celery worker configuration for testing +CELERY_TASK_ALWAYS_EAGER = True +CELERY_BROKER_URL = 'memory://' +# Celery worker configuration for production +# CELERY_TASK_ALWAYS_EAGER = False +# CELERY_BROKER_URL = 'redis://localhost:6379' + +# Celery settings, it is not recommended to change these +CELERY_WORKER_PREFETCH_MULTIPLIER = 0 +CELERY_BEAT_SCHEDULE_FILENAME = os.path.join( + DATA_DIR, 'celery', 'beat-schedule' +) + From 1decda965fa081bc1fd9ee280dec50089740ee58 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Mon, 15 Oct 2018 21:51:26 +0200 Subject: [PATCH 06/13] temp --- conf/settings_history/settings.3.2.0.py | 1 + scripts/install | 2 ++ 2 files changed, 3 insertions(+) diff --git a/conf/settings_history/settings.3.2.0.py b/conf/settings_history/settings.3.2.0.py index 8800bc5..e2e7b72 100644 --- a/conf/settings_history/settings.3.2.0.py +++ b/conf/settings_history/settings.3.2.0.py @@ -56,6 +56,7 @@ BASE_DIR = '__FINALPATH__' # Data directory DATA_DIR = os.path.join(BASE_DIR, 'data') +TTF_PATH = '/usr/share/fonts-droid-fallback/truetype/' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name diff --git a/scripts/install b/scripts/install index 6d99a0f..a8ebf42 100755 --- a/scripts/install +++ b/scripts/install @@ -198,6 +198,8 @@ virtualenv "${final_path}/venv" pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply # specific to YunoHost package: pip install django_sendmail_backend + # fix https://github.com/WeblateOrg/weblate/issues/2294 + pip install backports.csv ) #================================================= From 9f9814c2fa73109a9b02e5bc4bf981a98eeef743 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Tue, 16 Oct 2018 21:11:12 +0200 Subject: [PATCH 07/13] add TTF_PATH --- conf/settings_history/settings.3.2.0.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/settings_history/settings.3.2.0.py b/conf/settings_history/settings.3.2.0.py index e2e7b72..435b688 100644 --- a/conf/settings_history/settings.3.2.0.py +++ b/conf/settings_history/settings.3.2.0.py @@ -56,7 +56,7 @@ BASE_DIR = '__FINALPATH__' # Data directory DATA_DIR = os.path.join(BASE_DIR, 'data') -TTF_PATH = '/usr/share/fonts-droid-fallback/truetype/' +TTF_PATH = '__FINALPATH__/venv/lib/python2.7/site-packages/weblate/ttf/' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name From c38e8df3f46ee538b34184803cd713a8609cf460 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Tue, 16 Oct 2018 22:37:09 +0200 Subject: [PATCH 08/13] upgrade to 2.20 --- check_process | 6 +++--- .../settings_history/{settings.3.2.0.py => settings.3.2.py} | 0 manifest.json | 2 +- scripts/_common.sh | 2 +- scripts/upgrade | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename conf/settings_history/{settings.3.2.0.py => settings.3.2.py} (100%) diff --git a/check_process b/check_process index d5ea6db..a6aba11 100644 --- a/check_process +++ b/check_process @@ -19,7 +19,7 @@ setup_private=1 setup_public=1 upgrade=1 - # upgrade=1 from_commit=f2c1f14df484d727f1918821b6ade3887f77ec40 + upgrade=1 from_commit=f14080cbe65c8b2a8e0cda1c34affe0123a6540d backup_restore=1 multi_instance=1 incorrect_path=1 @@ -42,6 +42,6 @@ Email=jean-baptiste@holcroft.fr Notification=all ;;; Upgrade options - ; commit=f2c1f14df484d727f1918821b6ade3887f77ec40 - name=Upgrade from 2.20 + ; commit=f14080cbe65c8b2a8e0cda1c34affe0123a6540d + name=Upgrade from 3.1.1 manifest_arg=domain=DOMAIN&path=PATH&admin=USER&is_public=1&github_account=fake&github_token=fake diff --git a/conf/settings_history/settings.3.2.0.py b/conf/settings_history/settings.3.2.py similarity index 100% rename from conf/settings_history/settings.3.2.0.py rename to conf/settings_history/settings.3.2.py diff --git a/manifest.json b/manifest.json index 38e70e0..739fb0d 100644 --- a/manifest.json +++ b/manifest.json @@ -8,7 +8,7 @@ "description": { "en": "A translation platform using Git and Python" }, - "version": "3.2.0~ynh1", + "version": "3.2~ynh1", "url": "https://weblate.org", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 428c56c..6b10cee 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,6 +1,6 @@ #!/bin/bash -current_version="3.2.0" +current_version="3.2" ynh_check_global_uwsgi_config () { uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" diff --git a/scripts/upgrade b/scripts/upgrade index 1065456..a91a3c0 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -107,7 +107,7 @@ fi # Weblate requires an update to 3.0 before 3.1 # the upgrade hook will launch this script again to make sure it works -if [[ -z "$migration311" ]] +if [[ -z "$migration311" && $previous_version = "2.20" ]] then # $migration311 is not set, version is <3.0 migration311="two_steps_upgrade_3.0to3.1-needed" From 9cf6fd0e89d0829825a1a06ebd576db744123ca4 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 19 Oct 2018 23:28:40 +0200 Subject: [PATCH 09/13] migrate to python3, add celery and redis --- README.md | 2 -- conf/celery-weblate | 27 +++++++++++++++ conf/celery-weblate.service | 23 +++++++++++++ conf/settings_history/settings.3.2.py | 18 ++++++---- conf/uwsgi.ini | 4 +-- scripts/_common.sh | 47 ++++++++++++++++++++++++++- scripts/backup | 6 ++++ scripts/install | 36 +++++++++++++------- scripts/remove | 11 ++++++- scripts/restore | 16 ++++++--- scripts/upgrade | 14 ++++---- 11 files changed, 168 insertions(+), 36 deletions(-) create mode 100644 conf/celery-weblate create mode 100644 conf/celery-weblate.service diff --git a/README.md b/README.md index b4ba459..5e95f42 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,11 @@ It doesn't work yet, but while [it looks doable](https://docs.weblate.org/en/lat [ ] ARM support * to be added: -[ ] strenghen uwsgi systemd security (https://github.com/YunoHost-Apps/kresus_ynh/pull/21/commits/0ce4979d5037b111aa9698ed5552135bf0ed7165) [ ] use jq instead of grep/sed [ ] change URL script [ ] use debian package for lxml (may unlock ARM support) [ ] Add configuration options using the YunoHost interface (https://forum.yunohost.org/t/yunohost-3-1-minor-stable-release-version-stable-mineure/5445) [ ] Add fail2ban script -[ ] Use redis+hiredis for cache? [ ] Enable CHECK_LIST? [ ] Enable AUTOFIX_LIST? [ ] Enable Translation Memory? diff --git a/conf/celery-weblate b/conf/celery-weblate new file mode 100644 index 0000000..b1b3091 --- /dev/null +++ b/conf/celery-weblate @@ -0,0 +1,27 @@ +# Name of nodes to start +# here we have a single node +CELERYD_NODES="w1" +# or we could have three nodes: +#CELERYD_NODES="w1 w2 w3" + +# Absolute or relative path to the 'celery' command: +CELERY_BIN="/usr/bin/celery" + +# App instance to use +# comment out this line if you don't use an app +CELERY_APP="weblate" + +# How to call manage.py +CELERYD_MULTI="multi" + +# Extra command-line arguments to the worker +CELERYD_OPTS="--beat" + +# - %n will be replaced with the first part of the nodename. +# - %I will be replaced with the current child process index +# and is important when using the prefork pool to avoid race conditions. +CELERYD_PID_FILE="/var/run/celery-__APP__/weblate-%n.pid" +CELERYD_LOG_FILE="/var/log/uwsgi/app/__APP__-celery-%n%I.log" +CELERYD_LOG_LEVEL="INFO" + +CELERY_WORKER_RUNNING="1" \ No newline at end of file diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service new file mode 100644 index 0000000..59e566e --- /dev/null +++ b/conf/celery-weblate.service @@ -0,0 +1,23 @@ +[Unit] +Description=Celery Service for Weblate (__APP__) +After=network.target + +[Service] +Type=forking +User=__APP__ +Group=__APP__ +PermissionsStartOnly=true +EnvironmentFile=__FINALPATH__/celery-weblate +Environment=PATH=__FINALPATH__/venv/bin:$PATH +WorkingDirectory=__FINALPATH__/ +ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' +ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ + --pidfile=${CELERYD_PID_FILE}' +ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \ + -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/settings_history/settings.3.2.py b/conf/settings_history/settings.3.2.py index 435b688..d614042 100644 --- a/conf/settings_history/settings.3.2.py +++ b/conf/settings_history/settings.3.2.py @@ -56,7 +56,7 @@ BASE_DIR = '__FINALPATH__' # Data directory DATA_DIR = os.path.join(BASE_DIR, 'data') -TTF_PATH = '__FINALPATH__/venv/lib/python2.7/site-packages/weblate/ttf/' +TTF_PATH = '__FINALPATH__/venv/lib/python3.5/site-packages/weblate/ttf/' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name @@ -715,8 +715,12 @@ ALLOWED_HOSTS = ['__DOMAIN__'] CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:__MEMCPORT__', + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379/_REDIS_DB__', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + 'PARSER_CLASS': 'redis.connection.HiredisParser', + } }, 'avatar': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', @@ -806,11 +810,11 @@ REST_FRAMEWORK = { # ) # Celery worker configuration for testing -CELERY_TASK_ALWAYS_EAGER = True -CELERY_BROKER_URL = 'memory://' +# CELERY_TASK_ALWAYS_EAGER = True +# CELERY_BROKER_URL = 'memory://' # Celery worker configuration for production -# CELERY_TASK_ALWAYS_EAGER = False -# CELERY_BROKER_URL = 'redis://localhost:6379' +CELERY_TASK_ALWAYS_EAGER = False +CELERY_BROKER_URL = 'redis://localhost:6379/__REDIS_DB__' # Celery settings, it is not recommended to change these CELERY_WORKER_PREFETCH_MULTIPLIER = 0 diff --git a/conf/uwsgi.ini b/conf/uwsgi.ini index 2c3929e..aff091c 100644 --- a/conf/uwsgi.ini +++ b/conf/uwsgi.ini @@ -1,12 +1,12 @@ [uwsgi] -plugins = python +plugins = python3 master = true protocol = uwsgi socket = /var/run/uwsgi/__APP__.socket virtualenv = __FINALPATH__/venv # http://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#hosting-multiple-apps-in-the-same-process-aka-managing-script-name-and-path-info -mount = __PATH__=__FINALPATH__/venv/lib/python2.7/site-packages/weblate/wsgi.py +mount = __PATH__=__FINALPATH__/venv/lib/python3.5/site-packages/weblate/wsgi.py manage-script-name = true # Needed for OAuth/OpenID diff --git a/scripts/_common.sh b/scripts/_common.sh index 6b10cee..2c5e87c 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -91,8 +91,8 @@ weblate_fill_settings() { ynh_replace_string "__DOMAIN__" "$domain" "$settings" ynh_replace_string "__KEY__" "$key" "$settings" ynh_replace_string "__FINALPATH__" "$final_path" "$settings" - ynh_replace_string "__MEMCPORT__" "$memc_port" "$settings" ynh_replace_string "__GITHUBUSER__" "$github_account" "$settings" + ynh_replace_string "__REDIS_DB__" "$redis_db" "$settings" # root install as an empty PATHURL to prevent '//static' if [ "$path_url" == "/" ] @@ -177,3 +177,48 @@ $(yunohost tools diagnosis | grep -B 100 "services:" | sed '/services:/d')" # Send the email to the recipients echo "$mail_message" | $mail_bin -a "Content-Type: text/plain; charset=UTF-8" -s "$mail_subject" "$recipients" } + +#================================================= +# +# Redis HELPERS +# +# Point of contact : Jean-Baptiste Holcroft +#================================================= + +# get the first available redis database +# +# usage: ynh_redis_get_free_db +# | returns: the database number to use +ynh_redis_get_free_db() { + local result max db + result=$(redis-cli INFO keyspace) + + # get the num + max=$(cat /etc/redis/redis.conf | grep ^databases | grep -Eow "[0-9]+") + + db=0 + # default Debian setting is 15 databases + for i in $(seq 0 "$max") + do + if ! echo "$result" | grep -q "db$i" + then + db=$i + break 1 + fi + db=-1 + done + + test "$db" -eq -1 && ynh_die "No available Redis databases..." + + echo "$db" +} + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_redis_remove_db database +# | arg: database - the database to erase +ynh_redis_remove_db() { + local db=$1 + redis-cli -n "$db" flushall +} \ No newline at end of file diff --git a/scripts/backup b/scripts/backup index e159b99..3578fbe 100755 --- a/scripts/backup +++ b/scripts/backup @@ -56,6 +56,12 @@ ynh_backup "db.sql" ynh_backup "/etc/cron.d/$app" +#================================================= +# BACKUP THE CRON FILE +#================================================= + +ynh_backup "/usr/lib/tmpfiles.d/$app.conf" + #================================================= # BACKUP THE uwsgi files #================================================= diff --git a/scripts/install b/scripts/install index e3a6fce..d382546 100755 --- a/scripts/install +++ b/scripts/install @@ -77,9 +77,9 @@ ynh_app_setting_set "$app" github_token "$github_token" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 \ + mailutils python-celery-common virtualenv redis-server #================================================= # CREATE A PostgreSQL DATABASE @@ -185,7 +185,7 @@ fi #================================================= # PIP INSTALLATION #================================================= -virtualenv "${final_path}/venv" +virtualenv --python=python3 "${final_path}/venv" #run source in a 'sub shell' ( set +o nounset @@ -195,11 +195,9 @@ virtualenv "${final_path}/venv" # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis # specific to YunoHost package: pip install django_sendmail_backend - # Fix ImportError: No module named backports - pip install backports.csv ) #================================================= @@ -210,12 +208,13 @@ virtualenv "${final_path}/venv" db_pwd=$(ynh_app_setting_get "$app" psqlpwd) admin_mail=$(ynh_user_get_info "$admin" mail) key=$(ynh_string_random 24)$(ynh_string_random 24)$(ynh_string_random 2) -memc_port=$(ynh_find_port 8080) -settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +redis_db=$(ynh_redis_get_free_db) + +settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" cp "../conf/settings_history/settings.$current_version.py" "$settings" weblate_fill_settings "$settings" -ynh_app_setting_set "$app" memc_port "$memc_port" +ynh_app_setting_set "$app" redis_db "$redis_db" #================================================= # SPECIFIC SETUP Filling up the database @@ -249,7 +248,21 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app" #================================================= # Calculate and store the config file checksum into the app settings -ynh_store_file_checksum "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" + +#================================================= +# ACTIVATE CELERY +#================================================= + +celeryconf="$final_path/celery-weblate" +cp ../conf/celery-weblate "$celeryconf" + +ynh_replace_string "__APP__" "$app" "$celeryconf" + +echo "d /var/run/celery-$app 0775 $app $app" > "/usr/lib/tmpfiles.d/$app.conf" +systemd-tmpfiles --create + +ynh_add_systemd_config "$app-celery" "celery-weblate.service" #================================================= # GENERIC FINALIZATION @@ -287,6 +300,7 @@ fi # Start weblate #================================================= +systemctl start "$app-celery" systemctl start "uwsgi-app@$app.service" #================================================= diff --git a/scripts/remove b/scripts/remove index 841b4ff..6adfad6 100755 --- a/scripts/remove +++ b/scripts/remove @@ -18,10 +18,11 @@ domain=$(ynh_app_setting_get "$app" domain) db_name=$(ynh_app_setting_get "$app" db_name) #================================================= -# STOP WEBLATE'S SERVICE +# STOP WEBLATE'S SERVICES #================================================= systemctl stop "uwsgi-app@$app.service" +systemctl stop "$app-celery.service" #================================================= # REMOVE THE PostgreSQL DATABASE @@ -60,11 +61,19 @@ ynh_remove_nginx_config # Remove a cron file ynh_secure_remove "/etc/cron.d/$app" +#================================================= +# REMOVE TMPFILES.D +#================================================= + +ynh_secure_remove "/var/run/celery-$app" +ynh_secure_remove "/usr/lib/tmpfiles.d/$app.conf" + #================================================= # REMOVE uwsgi and systemd files #================================================= ynh_remove_uwsgi_service +ynh_remove_systemd_config "$app-celery" #================================================= # GENERIC FINALIZATION diff --git a/scripts/restore b/scripts/restore index a5518bd..ae0ea38 100755 --- a/scripts/restore +++ b/scripts/restore @@ -75,9 +75,9 @@ chown -R "$app": "$final_path" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \ + mailutils python-celery-common virtualenv redis-server #================================================= # RESTORE THE PostgreSQL DATABASE @@ -123,6 +123,13 @@ yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" ynh_restore_file "/etc/cron.d/$app" +#================================================= +# RESTORE THE TMPFILES.D +#================================================= + +ynh_restore_file "/usr/lib/tmpfiles.d/$app.conf" +systemd-tmpfiles --create + #================================================= # RESTORE THE HUB BINARY FILE #================================================= @@ -135,10 +142,11 @@ ynh_restore_file "/usr/bin/hub" # Start weblate #================================================= +systemctl start "$app-celery" systemctl start "uwsgi-app@$app.service" #================================================= # RELOAD NGINX & uwsgi #================================================= -systemctl reload nginx +systemctl reload nginx \ No newline at end of file diff --git a/scripts/upgrade b/scripts/upgrade index a91a3c0..90a2760 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -186,9 +186,9 @@ chsh --shell /bin/bash "$app" #================================================= ynh_install_app_dependencies libxml2-dev libxslt-dev libfreetype6-dev \ - libjpeg-dev libz-dev libyaml-dev python-dev python-pip python-virtualenv \ - postgresql libpq-dev uwsgi uwsgi-plugin-python memcached \ - mailutils + libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + postgresql libpq-dev uwsgi uwsgi-plugin-python3 memcached \ + mailutils python-celery-common virtualenv redis-server #================================================= # SPECIFIC SETUP uwsgi @@ -215,12 +215,10 @@ fi # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar pydns psycopg2-binary python-memcached phply + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis # specific to YunoHost package: pip install django_sendmail_backend - # Fix ImportError: No module named backports - pip install backports.csv -) +)) #================================================= # CONFIG FILE UPGRADE @@ -374,4 +372,4 @@ systemctl start "uwsgi-app@$app.service" # RELOAD NGINX #================================================= -systemctl reload nginx +systemctl reload nginx \ No newline at end of file From 08bd696434efe7eea0308330f1d1606757db0bb7 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 24 Oct 2018 22:08:30 +0200 Subject: [PATCH 10/13] Fix celery behavior --- conf/celery-weblate | 7 ++----- conf/celery-weblate.service | 17 ++++++++++------- .../{settings.3.2.py => settings.3.2.2.py} | 4 ++-- manifest.json | 2 +- scripts/_common.sh | 2 +- scripts/backup | 6 ------ scripts/install | 6 ++---- scripts/remove | 5 ++--- scripts/restore | 7 ------- 9 files changed, 20 insertions(+), 36 deletions(-) rename conf/settings_history/{settings.3.2.py => settings.3.2.2.py} (99%) diff --git a/conf/celery-weblate b/conf/celery-weblate index b1b3091..fc28813 100644 --- a/conf/celery-weblate +++ b/conf/celery-weblate @@ -4,9 +4,6 @@ CELERYD_NODES="w1" # or we could have three nodes: #CELERYD_NODES="w1 w2 w3" -# Absolute or relative path to the 'celery' command: -CELERY_BIN="/usr/bin/celery" - # App instance to use # comment out this line if you don't use an app CELERY_APP="weblate" @@ -20,8 +17,8 @@ CELERYD_OPTS="--beat" # - %n will be replaced with the first part of the nodename. # - %I will be replaced with the current child process index # and is important when using the prefork pool to avoid race conditions. -CELERYD_PID_FILE="/var/run/celery-__APP__/weblate-%n.pid" -CELERYD_LOG_FILE="/var/log/uwsgi/app/__APP__-celery-%n%I.log" +CELERYD_PID_FILE="/var/run/__APP__-celery/weblate-%n.pid" +CELERYD_LOG_FILE="/var/log/__APP__-celery/celery-%n%I.log" CELERYD_LOG_LEVEL="INFO" CELERY_WORKER_RUNNING="1" \ No newline at end of file diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service index 59e566e..87be424 100644 --- a/conf/celery-weblate.service +++ b/conf/celery-weblate.service @@ -8,16 +8,19 @@ User=__APP__ Group=__APP__ PermissionsStartOnly=true EnvironmentFile=__FINALPATH__/celery-weblate -Environment=PATH=__FINALPATH__/venv/bin:$PATH WorkingDirectory=__FINALPATH__/ -ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \ +ExecStartPre=/bin/mkdir -p /var/run/__APP__-celery +ExecStartPre=/bin/chown -R __APP__ /var/run/__APP__-celery +ExecStartPre=/bin/mkdir -p /var/log/__APP__-celery +ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__-celery +ExecStart=__FINALPATH__/venv/bin/celery multi start ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' -ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ - --pidfile=${CELERYD_PID_FILE}' -ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} +ExecStop=__FINALPATH__/venv/bin/celery multi stopwait ${CELERYD_NODES} \ + --pidfile=${CELERYD_PID_FILE} +ExecReload=__FINALPATH__/venv/bin/celery multi restart ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/conf/settings_history/settings.3.2.py b/conf/settings_history/settings.3.2.2.py similarity index 99% rename from conf/settings_history/settings.3.2.py rename to conf/settings_history/settings.3.2.2.py index d614042..ffbe881 100644 --- a/conf/settings_history/settings.3.2.py +++ b/conf/settings_history/settings.3.2.2.py @@ -716,7 +716,7 @@ ALLOWED_HOSTS = ['__DOMAIN__'] CACHES = { 'default': { 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://127.0.0.1:6379/_REDIS_DB__', + 'LOCATION': 'redis://127.0.0.1:6379/__REDIS_DB__', 'OPTIONS': { 'CLIENT_CLASS': 'django_redis.client.DefaultClient', 'PARSER_CLASS': 'redis.connection.HiredisParser', @@ -814,7 +814,7 @@ REST_FRAMEWORK = { # CELERY_BROKER_URL = 'memory://' # Celery worker configuration for production CELERY_TASK_ALWAYS_EAGER = False -CELERY_BROKER_URL = 'redis://localhost:6379/__REDIS_DB__' +CELERY_BROKER_URL = 'redis://127.0.0.1:6379/__REDIS_DB__' # Celery settings, it is not recommended to change these CELERY_WORKER_PREFETCH_MULTIPLIER = 0 diff --git a/manifest.json b/manifest.json index 739fb0d..eeee946 100644 --- a/manifest.json +++ b/manifest.json @@ -8,7 +8,7 @@ "description": { "en": "A translation platform using Git and Python" }, - "version": "3.2~ynh1", + "version": "3.2.2~ynh1", "url": "https://weblate.org", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 2c5e87c..e6bff98 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -1,6 +1,6 @@ #!/bin/bash -current_version="3.2" +current_version="3.2.2" ynh_check_global_uwsgi_config () { uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" diff --git a/scripts/backup b/scripts/backup index 3578fbe..e159b99 100755 --- a/scripts/backup +++ b/scripts/backup @@ -56,12 +56,6 @@ ynh_backup "db.sql" ynh_backup "/etc/cron.d/$app" -#================================================= -# BACKUP THE CRON FILE -#================================================= - -ynh_backup "/usr/lib/tmpfiles.d/$app.conf" - #================================================= # BACKUP THE uwsgi files #================================================= diff --git a/scripts/install b/scripts/install index d382546..c6236b8 100755 --- a/scripts/install +++ b/scripts/install @@ -257,10 +257,8 @@ ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/se celeryconf="$final_path/celery-weblate" cp ../conf/celery-weblate "$celeryconf" -ynh_replace_string "__APP__" "$app" "$celeryconf" - -echo "d /var/run/celery-$app 0775 $app $app" > "/usr/lib/tmpfiles.d/$app.conf" -systemd-tmpfiles --create +ynh_replace_string "__APP__" "$app" "$celeryconf" +ynh_replace_string "__FINALPATH__" "$final_path" "$celeryconf" ynh_add_systemd_config "$app-celery" "celery-weblate.service" diff --git a/scripts/remove b/scripts/remove index 6adfad6..b341da1 100755 --- a/scripts/remove +++ b/scripts/remove @@ -62,11 +62,10 @@ ynh_remove_nginx_config ynh_secure_remove "/etc/cron.d/$app" #================================================= -# REMOVE TMPFILES.D +# REMOVE CELERY FILES #================================================= -ynh_secure_remove "/var/run/celery-$app" -ynh_secure_remove "/usr/lib/tmpfiles.d/$app.conf" +ynh_secure_remove "/var/run/$app-celery" #================================================= # REMOVE uwsgi and systemd files diff --git a/scripts/restore b/scripts/restore index ae0ea38..d42fc52 100755 --- a/scripts/restore +++ b/scripts/restore @@ -123,13 +123,6 @@ yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" ynh_restore_file "/etc/cron.d/$app" -#================================================= -# RESTORE THE TMPFILES.D -#================================================= - -ynh_restore_file "/usr/lib/tmpfiles.d/$app.conf" -systemd-tmpfiles --create - #================================================= # RESTORE THE HUB BINARY FILE #================================================= From 1d182631b26ac3cb948f5e6925d04576af8cc947 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 25 Oct 2018 08:01:46 +0200 Subject: [PATCH 11/13] Fix backup and restore for celery --- scripts/backup | 1 + scripts/restore | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/backup b/scripts/backup index e159b99..a2eaffe 100755 --- a/scripts/backup +++ b/scripts/backup @@ -62,6 +62,7 @@ ynh_backup "/etc/cron.d/$app" ynh_backup "/etc/uwsgi/apps-available/$app.ini" ynh_backup "/etc/systemd/system/uwsgi-app@.service" +ynh_backup "/etc/systemd/system/$app-celery.service" #================================================= # BACKUP THE hub binary file diff --git a/scripts/restore b/scripts/restore index d42fc52..b8b5788 100755 --- a/scripts/restore +++ b/scripts/restore @@ -117,6 +117,8 @@ systemctl enable "uwsgi-app@$app.service" # Add as a service yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" +ynh_restore_file "/etc/systemd/system/$app-celery.service" + #================================================= # RESTORE THE CRON FILE #================================================= @@ -135,7 +137,7 @@ ynh_restore_file "/usr/bin/hub" # Start weblate #================================================= -systemctl start "$app-celery" +systemctl start "$app-celery.service" systemctl start "uwsgi-app@$app.service" #================================================= From d78dc9b9189dd46f8e01d9e9a734a6f745611343 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 25 Oct 2018 08:01:56 +0200 Subject: [PATCH 12/13] Fix upgrade for celery --- scripts/upgrade | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/upgrade b/scripts/upgrade index 90a2760..d73d15c 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -26,6 +26,7 @@ memc_port=$(ynh_app_setting_get "$app" memc_port) github_account=$(ynh_app_setting_get "$app" github_account) migration311=$(ynh_app_setting_get "$app" migration311) key=$(ynh_string_random 24)$(ynh_string_random 24)$(ynh_string_random 2) +redis_db=$(ynh_app_setting_get "$app" redis_db) #================================================= # Get previous version number @@ -63,7 +64,19 @@ if [ -z "$db_name" ]; then # If db_name doesn't exist, create it ynh_app_setting_set "$app" db_name "$db_name" fi -settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +if [ -e "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" ]; then + settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" + mv "$settings" "$final_path/settings.py" + ln -s "$final_path/settings.py" "$settings" +fi + +if [ -e "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" ]; then + settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" + mv "$settings" "$final_path/settings.py" + ln -s "$final_path/settings.py" "$settings" +fi + +settings="$final_path/settings.py" # (<2.17) save memc_port if it doesn't exist if [[ -z "$memc_port" ]] @@ -133,6 +146,17 @@ then systemctl stop "uwsgi-app@$app.service" fi +if [[ -e "/var/run/$app-celery/$app-w1.pid" ]] +then + systemctl stop "$app-celery.service" +fi + +# (<3.2) +if [ -z "$redis_db" ]; then + redis_db=$(ynh_redis_get_free_db) + ynh_app_setting_set "$app" redis_db "$redis_db" +fi + #================================================= # BACKUP BEFORE UPGRADE THEN ACTIVE TRAP #================================================= @@ -207,6 +231,8 @@ fi # PIP INSTALLATION #================================================= +ynh_secure_remove "${final_path}/venv" +virtualenv --python=python3 "${final_path}/venv" ( set +o nounset source "${final_path}/venv/bin/activate" @@ -218,7 +244,7 @@ fi pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis # specific to YunoHost package: pip install django_sendmail_backend -)) +) #================================================= # CONFIG FILE UPGRADE @@ -367,6 +393,7 @@ fi #================================================= systemctl start "uwsgi-app@$app.service" +systemctl start "$app-celery.service" #================================================= # RELOAD NGINX From cc02b3869016a6c694c476345010cbb29aaba179 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 26 Oct 2018 22:48:09 +0200 Subject: [PATCH 13/13] Fix upgrade --- scripts/upgrade | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/upgrade b/scripts/upgrade index d73d15c..767f4df 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -64,20 +64,16 @@ if [ -z "$db_name" ]; then # If db_name doesn't exist, create it ynh_app_setting_set "$app" db_name "$db_name" fi +# (<3.2) +migrate_to_python3=0 if [ -e "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" ]; then settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" - mv "$settings" "$final_path/settings.py" - ln -s "$final_path/settings.py" "$settings" -fi - -if [ -e "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" ]; then + migrate_to_python3=1 + cp "$settings" "$final_path/settings.py" +else settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" - mv "$settings" "$final_path/settings.py" - ln -s "$final_path/settings.py" "$settings" fi -settings="$final_path/settings.py" - # (<2.17) save memc_port if it doesn't exist if [[ -z "$memc_port" ]] then @@ -231,8 +227,12 @@ fi # PIP INSTALLATION #================================================= -ynh_secure_remove "${final_path}/venv" -virtualenv --python=python3 "${final_path}/venv" +if [ "$migrate_to_python3" -eq 1 ] +then + ynh_secure_remove "${final_path}/venv" + virtualenv --python=python3 "${final_path}/venv" +fi + ( set +o nounset source "${final_path}/venv/bin/activate" @@ -246,6 +246,12 @@ virtualenv --python=python3 "${final_path}/venv" pip install django_sendmail_backend ) +if [ "$migrate_to_python3" -eq 1 ] +then + mv "$final_path/settings.py" "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" + settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" +fi + #================================================= # CONFIG FILE UPGRADE #================================================= @@ -301,6 +307,18 @@ weblate_fill_settings "$settings" ynh_secure_remove "$old_settings" +#================================================= +# ACTIVATE CELERY +#================================================= + +celeryconf="$final_path/celery-weblate" +cp ../conf/celery-weblate "$celeryconf" + +ynh_replace_string "__APP__" "$app" "$celeryconf" +ynh_replace_string "__FINALPATH__" "$final_path" "$celeryconf" + +ynh_add_systemd_config "$app-celery" "celery-weblate.service" + #================================================= # Run migration scripts #================================================= @@ -346,7 +364,7 @@ ynh_secure_remove "$old_settings" ) # Recalculate and store the config file checksum into the app settings -ynh_store_file_checksum "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" +ynh_store_file_checksum "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" #================================================= # SETUP CRON