From 5b6ffa4bc2e46b951274aa831ddb40174d99429f Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 22 Aug 2019 22:29:05 +0200 Subject: [PATCH 01/14] update to 3.8 --- check_process | 4 + conf/settings_history/settings.3.8.py | 887 ++++++++++++++++++++++++++ manifest.json | 2 +- scripts/_common.sh | 2 +- 4 files changed, 893 insertions(+), 2 deletions(-) create mode 100644 conf/settings_history/settings.3.8.py diff --git a/check_process b/check_process index cd74ddb..33788ce 100644 --- a/check_process +++ b/check_process @@ -21,6 +21,7 @@ upgrade=1 upgrade=1 from_commit=565bfc66719f55b9c6a03cba8ac9991111992c9b upgrade=1 from_commit=73a808f7a93e063e652533244fa1657fbc41c4f3 + upgrade=1 from_commit=31b82950959699478f2372259116b6d117a85d69 backup_restore=1 multi_instance=1 incorrect_path=1 @@ -48,4 +49,7 @@ Notification=all manifest_arg=domain=DOMAIN&path=PATH&admin=USER&is_public=1&github_account=fake&github_token=fake ; commit=73a808f7a93e063e652533244fa1657fbc41c4f3 name=Upgrade from 3.6.1 +manifest_arg=domain=DOMAIN&path=PATH&admin=USER&is_public=1&github_account=fake&github_token=fake + ; commit=31b82950959699478f2372259116b6d117a85d69 + name=Upgrade from 3.7.0 manifest_arg=domain=DOMAIN&path=PATH&admin=USER&is_public=1&github_account=fake&github_token=fake diff --git a/conf/settings_history/settings.3.8.py b/conf/settings_history/settings.3.8.py new file mode 100644 index 0000000..4e4a608 --- /dev/null +++ b/conf/settings_history/settings.3.8.py @@ -0,0 +1,887 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2012 - 2019 Michal Čihař +# +# This file is part of Weblate +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from __future__ import unicode_literals + +import os +import platform +from logging.handlers import SysLogHandler + +# +# Django settings for Weblate project. +# + +DEBUG = True + +ADMINS = ( + # ('Your Name', 'your_email@example.com'), +) + +MANAGERS = ADMINS + +DATABASES = { + 'default': { + # Use 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + # Database name or path to database file if using sqlite3. + 'NAME': '__NAME__', + # Database user, not used with sqlite3. + 'USER': '__NAME__', + # Database password, not used with sqlite3. + 'PASSWORD': '__DB_PWD__', + # Set to empty string for localhost. Not used with sqlite3. + 'HOST': 'localhost', + # Set to empty string for default. Not used with sqlite3. + 'PORT': '5432', + # Customizations for databases + 'OPTIONS': { + # In case of using an older MySQL server, + # which has MyISAM as a default storage + # 'init_command': 'SET storage_engine=INNODB', + # Uncomment for MySQL older than 5.7: + # 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + # Set emoji capable charset for MySQL: + # 'charset': 'utf8mb4', + 'sslmode': 'disable' + }, + } +} + +BASE_DIR = '__FINALPATH__' + +# Data directory +DATA_DIR = os.path.join(BASE_DIR, 'data') +TTF_PATH = '__FINALPATH__/venv/lib/python3.5/site-packages/weblate/ttf/' + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# In a Windows environment this must be set to your system time zone. +TIME_ZONE = 'UTC' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +LANGUAGES = ( + ('ar', 'العربية'), + ('az', 'Azərbaycan'), + ('be', 'Беларуская'), + ('be@latin', 'Biełaruskaja'), + ('bg', 'Български'), + ('br', 'Brezhoneg'), + ('ca', 'Català'), + ('cs', 'Čeština'), + ('da', 'Dansk'), + ('de', 'Deutsch'), + ('en', 'English'), + ('el', 'Ελληνικά'), + ('en-gb', 'English (United Kingdom)'), + ('es', 'Español'), + ('fi', 'Suomi'), + ('fr', 'Français'), + ('gl', 'Galego'), + ('he', 'עברית'), + ('hu', 'Magyar'), + ('id', 'Indonesia'), + ('it', 'Italiano'), + ('ja', '日本語'), + ('kk', 'Қазақ тілі'), + ('ko', '한국어'), + ('nb', 'Norsk bokmål'), + ('nl', 'Nederlands'), + ('pl', 'Polski'), + ('pt', 'Português'), + ('pt-br', 'Português brasileiro'), + ('ru', 'Русский'), + ('sk', 'Slovenčina'), + ('sl', 'Slovenščina'), + ('sr', 'Српски'), + ('sv', 'Svenska'), + ('tr', 'Türkçe'), + ('uk', 'Українська'), + ('zh-hans', '简体字'), + ('zh-hant', '正體字'), +) + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale. +USE_L10N = True + +# If you set this to False, Django will not use timezone-aware datetimes. +USE_TZ = True + +# URL prefix to use, please see documentation for more details +URL_PREFIX = '__PATHURL__' + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_ROOT = os.path.join(DATA_DIR, 'media') + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '{0}/media/'.format(URL_PREFIX) + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = os.path.join(DATA_DIR, 'static') + +# URL prefix for static files. +# Example: "http://media.lawrence.com/static/" +STATIC_URL = '{0}/static/'.format(URL_PREFIX) + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'compressor.finders.CompressorFinder', +) + +# Make this unique, and don't share it with anybody. +# You can generate it using weblate/examples/generate-secret-key +SECRET_KEY = '__KEY__' # noqa + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + os.path.join(BASE_DIR, 'weblate', 'templates'), + ], + 'OPTIONS': { + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.request', + 'django.template.context_processors.csrf', + 'django.contrib.messages.context_processors.messages', + 'weblate.trans.context_processors.weblate_context', + ], + 'loaders': [ + ('django.template.loaders.cached.Loader', [ + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ]), + ], + }, + }, +] + + +# GitHub username for sending pull requests. +# Please see the documentation for more details. +GITHUB_USERNAME = "__GITHUBUSER__" + +# Authentication configuration +AUTHENTICATION_BACKENDS = ( + 'social_core.backends.email.EmailAuth', + # 'social_core.backends.google.GoogleOAuth2', + # 'social_core.backends.github.GithubOAuth2', + # 'social_core.backends.bitbucket.BitbucketOAuth', + 'social_core.backends.suse.OpenSUSEOpenId', + 'social_core.backends.ubuntu.UbuntuOpenId', + 'social_core.backends.fedora.FedoraOpenId', + # 'social_core.backends.facebook.FacebookOAuth2', + 'weblate.accounts.auth.WeblateUserBackend', +) + +# Custom user model +AUTH_USER_MODEL = 'weblate_auth.User' + +# Social auth backends setup +SOCIAL_AUTH_GITHUB_KEY = '' +SOCIAL_AUTH_GITHUB_SECRET = '' +SOCIAL_AUTH_GITHUB_SCOPE = ['user:email'] + +SOCIAL_AUTH_BITBUCKET_KEY = '' +SOCIAL_AUTH_BITBUCKET_SECRET = '' +SOCIAL_AUTH_BITBUCKET_VERIFIED_EMAILS_ONLY = True + +SOCIAL_AUTH_FACEBOOK_KEY = '' +SOCIAL_AUTH_FACEBOOK_SECRET = '' +SOCIAL_AUTH_FACEBOOK_SCOPE = ['email', 'public_profile'] +SOCIAL_AUTH_FACEBOOK_PROFILE_EXTRA_PARAMS = {'fields': 'id,name,email'} +SOCIAL_AUTH_FACEBOOK_API_VERSION = '3.1' + +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = '' +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = '' + +# Social auth settings +SOCIAL_AUTH_PIPELINE = ( + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'weblate.accounts.pipeline.store_params', + 'weblate.accounts.pipeline.verify_open', + 'social_core.pipeline.user.get_username', + 'weblate.accounts.pipeline.require_email', + 'social_core.pipeline.mail.mail_validation', + 'weblate.accounts.pipeline.revoke_mail_code', + 'weblate.accounts.pipeline.ensure_valid', + 'weblate.accounts.pipeline.remove_account', + 'social_core.pipeline.social_auth.associate_by_email', + 'weblate.accounts.pipeline.reauthenticate', + 'weblate.accounts.pipeline.verify_username', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'weblate.accounts.pipeline.cleanup_next', + 'weblate.accounts.pipeline.user_full_name', + 'weblate.accounts.pipeline.store_email', + 'weblate.accounts.pipeline.notify_connect', + 'weblate.accounts.pipeline.password_reset', +) +SOCIAL_AUTH_DISCONNECT_PIPELINE = ( + 'social_core.pipeline.disconnect.allowed_to_disconnect', + 'social_core.pipeline.disconnect.get_entries', + 'social_core.pipeline.disconnect.revoke_tokens', + 'weblate.accounts.pipeline.cycle_session', + 'weblate.accounts.pipeline.adjust_primary_mail', + 'weblate.accounts.pipeline.notify_disconnect', + 'social_core.pipeline.disconnect.disconnect', + 'weblate.accounts.pipeline.cleanup_next', +) + +# Custom authentication strategy +SOCIAL_AUTH_STRATEGY = 'weblate.accounts.strategy.WeblateStrategy' + +# Raise exceptions so that we can handle them later +SOCIAL_AUTH_RAISE_EXCEPTIONS = True + +SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = \ + 'weblate.accounts.pipeline.send_validation' +SOCIAL_AUTH_EMAIL_VALIDATION_URL = \ + '{0}/accounts/email-sent/'.format(URL_PREFIX) +SOCIAL_AUTH_LOGIN_ERROR_URL = \ + '{0}/accounts/login/'.format(URL_PREFIX) +SOCIAL_AUTH_EMAIL_FORM_URL = \ + '{0}/accounts/email/'.format(URL_PREFIX) +SOCIAL_AUTH_NEW_ASSOCIATION_REDIRECT_URL = \ + '{0}/accounts/profile/#account'.format(URL_PREFIX) +SOCIAL_AUTH_PROTECTED_USER_FIELDS = ('email',) +SOCIAL_AUTH_SLUGIFY_USERNAMES = True +SOCIAL_AUTH_SLUGIFY_FUNCTION = 'weblate.accounts.pipeline.slugify_username' + +# Password validation configuration +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa: E501, pylint: disable=line-too-long + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + 'OPTIONS': { + 'min_length': 6, + } + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, + { + 'NAME': 'weblate.accounts.password_validation.CharsPasswordValidator', + }, + { + 'NAME': 'weblate.accounts.password_validation.PastPasswordsValidator', + }, + # Optional password strength validation by django-zxcvbn-password + # { + # 'NAME': 'zxcvbn_password.ZXCVBNValidator', + # 'OPTIONS': { + # 'min_score': 3, + # 'user_attributes': ('username', 'email', 'full_name') + # } + # }, +] + +# Allow new user registrations +REGISTRATION_OPEN = True + +# Middleware +MIDDLEWARE = [ + 'weblate.middleware.ProxyMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'weblate.accounts.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'social_django.middleware.SocialAuthExceptionMiddleware', + 'weblate.accounts.middleware.RequireLoginMiddleware', + 'weblate.middleware.SecurityMiddleware', +] + +ROOT_URLCONF = 'weblate.urls' + +# Django and Weblate apps +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin.apps.SimpleAdminConfig', + 'django.contrib.admindocs', + 'django.contrib.sitemaps', + 'django.contrib.humanize', + 'social_django', + 'crispy_forms', + 'compressor', + 'rest_framework', + 'rest_framework.authtoken', + 'weblate.addons', + 'weblate.auth', + 'weblate.checks', + 'weblate.formats', + 'weblate.machinery', + 'weblate.trans', + 'weblate.lang', + 'weblate.langdata', + 'weblate.memory', + 'weblate.screenshots', + 'weblate.fonts', + 'weblate.accounts', + 'weblate.utils', + 'weblate.vcs', + 'weblate.wladmin', + 'weblate', + + # Optional: Git exporter + # 'weblate.gitexport', +) + +# Path to locales +LOCALE_PATHS = (os.path.join(BASE_DIR, 'weblate', 'locale'), ) + +# Custom exception reporter to include some details +DEFAULT_EXCEPTION_REPORTER_FILTER = \ + 'weblate.trans.debug.WeblateExceptionReporterFilter' + +# Default logging of Weblate messages +# - to syslog in production (if available) +# - otherwise to console +# - you can also choose 'logfile' to log into separate file +# after configuring it below + +# Detect if we can connect to syslog +HAVE_SYSLOG = False +if platform.system() != 'Windows': + try: + handler = SysLogHandler( + address='/dev/log', facility=SysLogHandler.LOG_LOCAL2 + ) + handler.close() + HAVE_SYSLOG = True + except IOError: + HAVE_SYSLOG = False + +if DEBUG or not HAVE_SYSLOG: + DEFAULT_LOG = 'console' +else: + DEFAULT_LOG = 'syslog' + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error when DEBUG=False. +# See http://docs.djangoproject.com/en/stable/topics/logging for +# more details on how to customize your logging configuration. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'syslog': { + 'format': 'weblate[%(process)d]: %(levelname)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + 'logfile': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'django.server': { + '()': 'django.utils.log.ServerFormatter', + 'format': '[%(server_time)s] %(message)s', + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler', + 'include_html': True, + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'django.server': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'django.server', + }, + 'syslog': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'formatter': 'syslog', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL2, + }, + # Logging to a file + # 'logfile': { + # 'level':'DEBUG', + # 'class':'logging.handlers.RotatingFileHandler', + # 'filename': "/var/log/weblate/weblate.log", + # 'maxBytes': 100000, + # 'backupCount': 3, + # 'formatter': 'logfile', + # }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['mail_admins', DEFAULT_LOG], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.server': { + 'handlers': ['django.server'], + 'level': 'INFO', + 'propagate': False, + }, + # Logging database queries + # 'django.db.backends': { + # 'handlers': [DEFAULT_LOG], + # 'level': 'DEBUG', + # }, + 'weblate': { + 'handlers': [DEFAULT_LOG], + 'level': 'DEBUG', + }, + # Logging search operations + 'weblate.search': { + 'handlers': [DEFAULT_LOG], + 'level': 'INFO', + }, + # Logging VCS operations + 'weblate.vcs': { + 'handlers': [DEFAULT_LOG], + 'level': 'WARNING', + }, + # Python Social Auth logging + # 'social': { + # 'handlers': [DEFAULT_LOG], + # 'level': 'DEBUG', + # }, + } +} + +# Logging of management commands to console +if (os.environ.get('DJANGO_IS_MANAGEMENT_COMMAND', False) + and 'console' not in LOGGING['loggers']['weblate']['handlers']): + LOGGING['loggers']['weblate']['handlers'].append('console') + +# Remove syslog setup if it's not present +if not HAVE_SYSLOG: + del LOGGING['handlers']['syslog'] + +# List of machine translations +# MT_SERVICES = ( +# 'weblate.machinery.apertium.ApertiumAPYTranslation', +# 'weblate.machinery.baidu.BaiduTranslation', +# 'weblate.machinery.deepl.DeepLTranslation', +# 'weblate.machinery.glosbe.GlosbeTranslation', +# 'weblate.machinery.google.GoogleTranslation', +# 'weblate.machinery.microsoft.MicrosoftCognitiveTranslation', +# 'weblate.machinery.microsoftterminology.MicrosoftTerminologyService', +# 'weblate.machinery.mymemory.MyMemoryTranslation', +# 'weblate.machinery.netease.NeteaseSightTranslation', +# 'weblate.machinery.tmserver.AmagamaTranslation', +# 'weblate.machinery.tmserver.TMServerTranslation', +# 'weblate.machinery.yandex.YandexTranslation', +# 'weblate.machinery.weblatetm.WeblateTranslation', +# 'weblate.machinery.saptranslationhub.SAPTranslationHub', +# 'weblate.machinery.youdao.YoudaoTranslation', +# 'weblate.memory.machine.WeblateMemory', +# ) + +# Machine translation API keys + +# URL of the Apertium APy server +MT_APERTIUM_APY = None + +# DeepL API key +MT_DEEPL_KEY = None + +# Microsoft Cognitive Services Translator API, register at +# https://portal.azure.com/ +MT_MICROSOFT_COGNITIVE_KEY = None + +# MyMemory identification email, see +# https://mymemory.translated.net/doc/spec.php +MT_MYMEMORY_EMAIL = None + +# Optional MyMemory credentials to access private translation memory +MT_MYMEMORY_USER = None +MT_MYMEMORY_KEY = None + +# Google API key for Google Translate API +MT_GOOGLE_KEY = None + +# Baidu app key and secret +MT_BAIDU_ID = None +MT_BAIDU_SECRET = None + +# Youdao Zhiyun app key and secret +MT_YOUDAO_ID = None +MT_YOUDAO_SECRET = None + +# Netease Sight (Jianwai) app key and secret +MT_NETEASE_KEY = None +MT_NETEASE_SECRET = None + +# API key for Yandex Translate API +MT_YANDEX_KEY = None + +# tmserver URL +MT_TMSERVER = None + +# SAP Translation Hub +MT_SAP_BASE_URL = None +MT_SAP_SANDBOX_APIKEY = None +MT_SAP_USERNAME = None +MT_SAP_PASSWORD = None +MT_SAP_USE_MT = True + +# Title of site to use +SITE_TITLE = 'Weblate' + +# Whether site uses https +ENABLE_HTTPS = True + +# Use HTTPS when creating redirect URLs for social authentication, see +# documentation for more details: +# https://python-social-auth-docs.readthedocs.io/en/latest/configuration/settings.html#processing-redirects-and-urlopen +SOCIAL_AUTH_REDIRECT_IS_HTTPS = ENABLE_HTTPS + +# Make CSRF cookie HttpOnly, see documentation for more details: +# https://docs.djangoproject.com/en/1.11/ref/settings/#csrf-cookie-httponly +CSRF_COOKIE_HTTPONLY = True +CSRF_COOKIE_SECURE = ENABLE_HTTPS +# Store CSRF token in session +CSRF_USE_SESSIONS = True +# Customize CSRF failure view +CSRF_FAILURE_VIEW = 'weblate.trans.views.error.csrf_failure' +SESSION_COOKIE_SECURE = ENABLE_HTTPS +# SSL redirect +SECURE_SSL_REDIRECT = ENABLE_HTTPS +# SSL redirect URL exemption list +SECURE_REDIRECT_EXEMPT = ( + r'healthz/$', # Allowing HTTP access to health check +) +# Session cookie age (in seconds) +SESSION_COOKIE_AGE = 1209600 + +# Some security headers +SECURE_BROWSER_XSS_FILTER = True +X_FRAME_OPTIONS = 'DENY' +SECURE_CONTENT_TYPE_NOSNIFF = True + +# Optionally enable HSTS +SECURE_HSTS_SECONDS = 0 +SECURE_HSTS_PRELOAD = False +SECURE_HSTS_INCLUDE_SUBDOMAINS = False + +# URL of login +LOGIN_URL = '{0}/accounts/login/'.format(URL_PREFIX) + +# URL of logout +LOGOUT_URL = '{0}/accounts/logout/'.format(URL_PREFIX) + +# Default location for login +LOGIN_REDIRECT_URL = '{0}/'.format(URL_PREFIX) + +# Anonymous user name +ANONYMOUS_USER_NAME = 'anonymous' + +# Reverse proxy settings +IP_PROXY_HEADER = 'HTTP_X_FORWARDED_FOR' +IP_BEHIND_REVERSE_PROXY = False +IP_PROXY_OFFSET = 0 + +# Sending HTML in mails +EMAIL_SEND_HTML = True + +# Subject of emails includes site title +EMAIL_SUBJECT_PREFIX = '[{0}] '.format(SITE_TITLE) + +EMAIL_BACKEND = 'django_sendmail_backend.backends.EmailBackend' + +# Enable remote hooks +ENABLE_HOOKS = True + +# Number of nearby messages to show in each direction +NEARBY_MESSAGES = 5 + +# By default the length of a given translation is limited to the length of +# the source string * 10 characters. Set this option to False to allow longer +# translations (up to 10.000 characters) +LIMIT_TRANSLATION_LENGTH_BY_SOURCE_LENGTH = True + +# Use simple language codes for default language/country combinations +SIMPLIFY_LANGUAGES = True + +# Render forms using bootstrap +CRISPY_TEMPLATE_PACK = 'bootstrap3' + +# List of quality checks +# CHECK_LIST = ( +# 'weblate.checks.same.SameCheck', +# 'weblate.checks.chars.BeginNewlineCheck', +# 'weblate.checks.chars.EndNewlineCheck', +# 'weblate.checks.chars.BeginSpaceCheck', +# 'weblate.checks.chars.EndSpaceCheck', +# 'weblate.checks.chars.EndStopCheck', +# 'weblate.checks.chars.EndColonCheck', +# 'weblate.checks.chars.EndQuestionCheck', +# 'weblate.checks.chars.EndExclamationCheck', +# 'weblate.checks.chars.EndEllipsisCheck', +# 'weblate.checks.chars.EndSemicolonCheck', +# 'weblate.checks.chars.MaxLengthCheck', +# 'weblate.checks.chars.KashidaCheck', +# 'weblate.checks.format.PythonFormatCheck', +# 'weblate.checks.format.PythonBraceFormatCheck', +# 'weblate.checks.format.PHPFormatCheck', +# 'weblate.checks.format.CFormatCheck', +# 'weblate.checks.format.PerlFormatCheck', +# 'weblate.checks.format.JavaScriptFormatCheck', +# 'weblate.checks.format.CSharpFormatCheck', +# 'weblate.checks.format.JavaFormatCheck', +# 'weblate.checks.format.JavaMessageFormatCheck', +# 'weblate.checks.angularjs.AngularJSInterpolationCheck', +# 'weblate.checks.qt.QtFormatCheck', +# 'weblate.checks.qt.QtPluralCheck', +# 'weblate.checks.ruby.RubyFormatCheck', +# 'weblate.checks.consistency.PluralsCheck', +# 'weblate.checks.consistency.SamePluralsCheck', +# 'weblate.checks.consistency.ConsistencyCheck', +# 'weblate.checks.consistency.TranslatedCheck', +# 'weblate.checks.chars.NewlineCountingCheck', +# 'weblate.checks.markup.BBCodeCheck', +# 'weblate.checks.chars.ZeroWidthSpaceCheck', +# 'weblate.checks.render.MaxSizeCheck', +# 'weblate.checks.markup.XMLValidityCheck', +# 'weblate.checks.markup.XMLTagsCheck', +# 'weblate.checks.markup.MarkdownRefLinkCheck', +# 'weblate.checks.markup.MarkdownLinkCheck', +# 'weblate.checks.markup.MarkdownSyntaxCheck', +# 'weblate.checks.markup.URLCheck', +# 'weblate.checks.source.OptionalPluralCheck', +# 'weblate.checks.source.EllipsisCheck', +# 'weblate.checks.source.MultipleFailingCheck', +# ) + +# List of automatic fixups +# AUTOFIX_LIST = ( +# 'weblate.trans.autofixes.whitespace.SameBookendingWhitespace', +# 'weblate.trans.autofixes.chars.ReplaceTrailingDotsWithEllipsis', +# 'weblate.trans.autofixes.chars.RemoveZeroSpace', +# 'weblate.trans.autofixes.chars.RemoveControlChars', +# ) + +# List of enabled addons +# WEBLATE_ADDONS = ( +# 'weblate.addons.gettext.GenerateMoAddon', +# 'weblate.addons.gettext.UpdateLinguasAddon', +# 'weblate.addons.gettext.UpdateConfigureAddon', +# 'weblate.addons.gettext.MsgmergeAddon', +# 'weblate.addons.gettext.GettextCustomizeAddon', +# 'weblate.addons.gettext.GettextAuthorComments', +# 'weblate.addons.cleanup.CleanupAddon', +# 'weblate.addons.consistency.LangaugeConsistencyAddon', +# 'weblate.addons.discovery.DiscoveryAddon', +# 'weblate.addons.flags.SourceEditAddon', +# 'weblate.addons.flags.TargetEditAddon', +# 'weblate.addons.flags.SameEditAddon', +# 'weblate.addons.generate.GenerateFileAddon', +# 'weblate.addons.json.JSONCustomizeAddon', +# 'weblate.addons.properties.PropertiesSortAddon', +# 'weblate.addons.git.GitSquashAddon', +# 'weblate.addons.removal.RemoveComments', +# 'weblate.addons.removal.RemoveSuggestions', +# ) + +# E-mail address that error messages come from. +SERVER_EMAIL = 'noreply@__DOMAIN__' + +# Default email address to use for various automated correspondence from +# the site managers. Used for registration emails. +DEFAULT_FROM_EMAIL = '__ADMINMAIL__' + +# List of URLs your site is supposed to serve +ALLOWED_HOSTS = ['__DOMAIN__'] + +CACHES = { + 'default': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379/__REDIS_DB__', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + 'PARSER_CLASS': 'redis.connection.HiredisParser', + } + }, + 'avatar': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': os.path.join(BASE_DIR, 'avatar-cache'), + 'TIMEOUT': 3600, + 'OPTIONS': { + 'MAX_ENTRIES': 1000, + }, + } + } + +# Example configuration for caching +# CACHES = { +# 'default': { +# 'BACKEND': 'django_redis.cache.RedisCache', +# 'LOCATION': 'redis://127.0.0.1:6379/0', +# # If redis is running on same host as Weblate, you might +# # want to use unix sockets instead: +# # 'LOCATION': 'unix:///var/run/redis/redis.sock?db=0', +# 'OPTIONS': { +# 'CLIENT_CLASS': 'django_redis.client.DefaultClient', +# 'PARSER_CLASS': 'redis.connection.HiredisParser', +# } +# }, +# 'avatar': { +# 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', +# 'LOCATION': os.path.join(DATA_DIR, 'avatar-cache'), +# 'TIMEOUT': 3600, +# 'OPTIONS': { +# 'MAX_ENTRIES': 1000, +# }, +# } +# } + +# REST framework settings for API +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + 'DEFAULT_PERMISSION_CLASSES': [ + 'rest_framework.permissions.IsAuthenticatedOrReadOnly' + ], + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework.authentication.TokenAuthentication', + 'weblate.api.authentication.BearerAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ), + 'DEFAULT_THROTTLE_CLASSES': ( + 'rest_framework.throttling.AnonRateThrottle', + 'rest_framework.throttling.UserRateThrottle' + ), + 'DEFAULT_THROTTLE_RATES': { + 'anon': '100/day', + 'user': '1000/day' + }, + 'DEFAULT_PAGINATION_CLASS': ( + 'rest_framework.pagination.PageNumberPagination' + ), + 'PAGE_SIZE': 20, + 'VIEW_DESCRIPTION_FUNCTION': 'weblate.api.views.get_view_description', + 'UNAUTHENTICATED_USER': 'weblate.auth.models.get_anonymous', +} + +# Example for restricting access to logged in users +# LOGIN_REQUIRED_URLS = ( +# r'/(.*)$', +# ) + +# In such case you will want to include some of the exceptions +# LOGIN_REQUIRED_URLS_EXCEPTIONS = ( +# r'/accounts/(.*)$', # Required for login +# r'/admin/login/(.*)$', # Required for admin login +# r'/static/(.*)$', # Required for development mode +# r'/widgets/(.*)$', # Allowing public access to widgets +# r'/data/(.*)$', # Allowing public access to data exports +# r'/hooks/(.*)$', # Allowing public access to notification hooks +# r'/healthz/$', # Allowing public access to health check +# r'/api/(.*)$', # Allowing access to API +# r'/js/i18n/$', # JavaScript localization +# r'/contact/$', # Optional for contact form +# r'/legal/(.*)$', # Optional for legal app +# ) + +# Silence some of the Django system checks +SILENCED_SYSTEM_CHECKS = [ + # We have modified django.contrib.auth.middleware.AuthenticationMiddleware + # as weblate.accounts.middleware.AuthenticationMiddleware + 'admin.E408', +] + +# Celery worker configuration for testing +# CELERY_TASK_ALWAYS_EAGER = True +# CELERY_BROKER_URL = 'memory://' +# CELERY_TASK_EAGER_PROPAGATES = True +# Celery worker configuration for production +CELERY_TASK_ALWAYS_EAGER = False +CELERY_BROKER_URL = 'redis://127.0.0.1:6379/__REDIS_DB__' +CELERY_RESULT_BACKEND = CELERY_BROKER_URL + +# Celery settings, it is not recommended to change these +CELERY_WORKER_PREFETCH_MULTIPLIER = 0 +CELERY_WORKER_MAX_MEMORY_PER_CHILD = 200000 +CELERY_BEAT_SCHEDULE_FILENAME = os.path.join( + DATA_DIR, 'celery', 'beat-schedule' +) +CELERY_TASK_ROUTES = { + 'weblate.trans.search.*': {'queue': 'search'}, + 'weblate.trans.tasks.optimize_fulltext': {'queue': 'search'}, + 'weblate.trans.tasks.cleanup_fulltext': {'queue': 'search'}, + 'weblate.memory.tasks.*': {'queue': 'memory'}, + 'weblate.accounts.tasks.notify_change': {'queue': 'notify'}, + 'weblate.accounts.tasks.send_mails': {'queue': 'notify'}, +} + diff --git a/manifest.json b/manifest.json index 5621cd9..0a8693f 100644 --- a/manifest.json +++ b/manifest.json @@ -8,7 +8,7 @@ "description": { "en": "A translation platform using Git and Python" }, - "version": "3.7.0~ynh2", + "version": "3.8.0~ynh1", "url": "https://weblate.org", "license": "AGPL-3.0", "maintainer": { diff --git a/scripts/_common.sh b/scripts/_common.sh index 02e8366..805b9c1 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -12,7 +12,7 @@ pkg_dependencies="libxml2-dev libxslt-dev libfreetype6-dev \ mailutils python-celery-common virtualenv redis-server" # Weblate's version for PIP and settings file -current_version="3.7" +current_version="3.8" #================================================= # PERSONAL HELPERS From 8bd9e233725c9e900279217b20dfb51b1872fc7d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 22 Aug 2019 22:29:38 +0200 Subject: [PATCH 02/14] remove old hook --- hooks/post_app_upgrade | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 hooks/post_app_upgrade diff --git a/hooks/post_app_upgrade b/hooks/post_app_upgrade deleted file mode 100644 index e397d3c..0000000 --- a/hooks/post_app_upgrade +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -migration311=$(yunohost app setting "$YNH_APP_INSTANCE_NAME" "migration311" --output-as plain --quiet) -mig_type=$(yunohost app setting "$YNH_APP_INSTANCE_NAME" "migration311_type" --output-as plain --quiet) -mig_parm=$(yunohost app setting "$YNH_APP_INSTANCE_NAME" "migration311_parm" --output-as plain --quiet) - -if [[ "$migration311" = "two_steps_upgrade_3.0to3.1-needed" ]] -then - yunohost app setting "$YNH_APP_INSTANCE_NAME" "migration311" --value="two_steps_upgrade_3.0to3.1-progress" --quiet - if [ "$mig_type" = "file" ] - then - yunohost app upgrade "$YNH_APP_INSTANCE_NAME" --file "$mig_parm" - else - yunohost app upgrade "$YNH_APP_INSTANCE_NAME" --url "$mig_parm" - fi - - yunohost app setting "$YNH_APP_INSTANCE_NAME" "migration311" --value="two_steps_upgrade_3.0to3.1-done" --quiet -fi \ No newline at end of file From 6b81c1ddcc6c563d08c718b6630230057c9a82c3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 22 Aug 2019 22:30:09 +0200 Subject: [PATCH 03/14] Disable DEBUG in production --- conf/settings_history/settings.3.8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/settings_history/settings.3.8.py b/conf/settings_history/settings.3.8.py index 4e4a608..354e703 100644 --- a/conf/settings_history/settings.3.8.py +++ b/conf/settings_history/settings.3.8.py @@ -28,7 +28,7 @@ from logging.handlers import SysLogHandler # Django settings for Weblate project. # -DEBUG = True +DEBUG = False ADMINS = ( # ('Your Name', 'your_email@example.com'), From 9a26ad6ab59c3152ab4464fbd17cbead17b772f3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 22 Aug 2019 22:31:15 +0200 Subject: [PATCH 04/14] Fix Admin email --- conf/settings_history/settings.3.8.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/settings_history/settings.3.8.py b/conf/settings_history/settings.3.8.py index 354e703..e2a22fc 100644 --- a/conf/settings_history/settings.3.8.py +++ b/conf/settings_history/settings.3.8.py @@ -31,7 +31,7 @@ from logging.handlers import SysLogHandler DEBUG = False ADMINS = ( - # ('Your Name', 'your_email@example.com'), + ('__ADMIN__', '__ADMINMAIL__'), ) MANAGERS = ADMINS From 5df42eb737e6a647e403c492e3e4fcd20aa54305 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Thu, 22 Aug 2019 22:34:59 +0200 Subject: [PATCH 05/14] add dep for subtitle translation support --- scripts/install | 2 +- scripts/upgrade | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install b/scripts/install index 6e3b2e0..820e5c1 100755 --- a/scripts/install +++ b/scripts/install @@ -189,7 +189,7 @@ virtualenv --python=python3 "${final_path}/venv" # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon # specific to YunoHost package: pip install django_sendmail_backend ) diff --git a/scripts/upgrade b/scripts/upgrade index 2bfb5d8..55dad7b 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -262,7 +262,7 @@ fi # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon # specific to YunoHost package: pip install django_sendmail_backend ) From 6872223e5ad4cadaa3f4ad2e8900b3c1e1f46362 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 23 Aug 2019 08:32:02 +0200 Subject: [PATCH 06/14] add logrotate --- conf/weblate.service | 2 +- scripts/_common.sh | 1 + scripts/backup | 7 +++++++ scripts/install | 8 ++++++++ scripts/remove | 8 ++++++++ scripts/restore | 6 ++++++ scripts/upgrade | 8 ++++++++ 7 files changed, 39 insertions(+), 1 deletion(-) diff --git a/conf/weblate.service b/conf/weblate.service index e9a947d..9b3d597 100644 --- a/conf/weblate.service +++ b/conf/weblate.service @@ -30,4 +30,4 @@ StandardError=syslog NotifyAccess=all [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/scripts/_common.sh b/scripts/_common.sh index 805b9c1..c8f2733 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -7,6 +7,7 @@ # dependencies used by the app pkg_dependencies="libxml2-dev libxslt-dev libfreetype6-dev \ libjpeg-dev libz-dev libyaml-dev python3-dev python3-pip python3-virtualenv \ + python3-enchant \ postgresql libpq-dev uwsgi uwsgi-plugin-python3 \ libpango1.0-dev libcairo2-dev libglib2.0-dev libgirepository1.0-dev \ mailutils python-celery-common virtualenv redis-server" diff --git a/scripts/backup b/scripts/backup index e357c7f..3f407e7 100755 --- a/scripts/backup +++ b/scripts/backup @@ -61,6 +61,13 @@ ynh_backup "db.sql" #================================================= # SPECIFIC BACKUP +#================================================= +# BACKUP LOGROTATE +#================================================= +ynh_script_progression --message="Backing up logrotate configuration..." --time --weight=1 + +ynh_backup --src_path="/etc/logrotate.d/$app" + #================================================= # BACKUP THE CRON FILE #================================================= diff --git a/scripts/install b/scripts/install index 820e5c1..402c2f4 100755 --- a/scripts/install +++ b/scripts/install @@ -300,6 +300,14 @@ chown -R "$app": "$final_path/data" mkdir -p "$final_path/avatar-cache" chown -R "$app": "$final_path/avatar-cache" +#================================================= +# SETUP LOGROTATE +#================================================= +ynh_script_progression --message="Configuring log rotation..." --time --weight=1 + +# Use logrotate to manage application logfile(s) +ynh_use_logrotate --logfile "/var/log/$app-celery" + #================================================= # SETUP SSOWAT #================================================= diff --git a/scripts/remove b/scripts/remove index 16be183..87adac0 100755 --- a/scripts/remove +++ b/scripts/remove @@ -79,6 +79,14 @@ ynh_script_progression --message="Removing nginx web server configuration..." -- # Remove the dedicated nginx config ynh_remove_nginx_config +#================================================= +# REMOVE LOGROTATE CONFIGURATION +#================================================= +ynh_script_progression --message="Removing logrotate configuration..." --time --weight=1 + +# Remove the app-specific logrotate config +ynh_remove_logrotate + #================================================= # SPECIFIC REMOVE #================================================= diff --git a/scripts/restore b/scripts/restore index ddbf231..fe52137 100755 --- a/scripts/restore +++ b/scripts/restore @@ -123,6 +123,12 @@ ynh_restore_file "/etc/cron.d/$app" ynh_script_progression --message="Restore hub's binary file..." --time --weight=1 ynh_restore_file "/usr/bin/hub" +#================================================= +# RESTORE THE LOGROTATE CONFIGURATION +#================================================= + +ynh_restore_file --origin_path="/etc/logrotate.d/$app" + #================================================= # GENERIC FINALIZATION #================================================= diff --git a/scripts/upgrade b/scripts/upgrade index 55dad7b..c5a9e9e 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -397,6 +397,14 @@ cp ../conf/cron "/etc/cron.d/$app" ynh_replace_string "__APP__" "$app" "/etc/cron.d/$app" ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app" +#================================================= +# SETUP LOGROTATE +#================================================= +ynh_script_progression --message="Upgrading logrotate configuration..." --time --weight=1 + +# Use logrotate to manage app-specific logfile(s) +ynh_use_logrotate --logfile "/var/log/$app-celery" --non-append + #================================================= # GENERIC FINALIZATION #================================================= From f05f984efeecc09ab48d98d89f1198f0d83343e2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 28 Aug 2019 00:09:23 +0200 Subject: [PATCH 07/14] improve log files management --- conf/celery-weblate | 6 +++--- conf/celery-weblate.service | 5 ++--- conf/weblate.service | 5 ++--- scripts/install | 6 +++--- scripts/restore | 4 ++-- scripts/upgrade | 11 ++++++++++- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/conf/celery-weblate b/conf/celery-weblate index fc28813..7b2d738 100644 --- a/conf/celery-weblate +++ b/conf/celery-weblate @@ -17,8 +17,8 @@ CELERYD_OPTS="--beat" # - %n will be replaced with the first part of the nodename. # - %I will be replaced with the current child process index # and is important when using the prefork pool to avoid race conditions. -CELERYD_PID_FILE="/var/run/__APP__-celery/weblate-%n.pid" -CELERYD_LOG_FILE="/var/log/__APP__-celery/celery-%n%I.log" +CELERYD_PID_FILE="/var/run/__APP__/weblate-celery-%n.pid" +CELERYD_LOG_FILE="/var/log/__APP__/weblate-celery-%n%I.log" CELERYD_LOG_LEVEL="INFO" -CELERY_WORKER_RUNNING="1" \ No newline at end of file +CELERY_WORKER_RUNNING="1" diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service index a399a1b..a6fe170 100644 --- a/conf/celery-weblate.service +++ b/conf/celery-weblate.service @@ -10,10 +10,9 @@ PermissionsStartOnly=true EnvironmentFile=__FINALPATH__/celery-weblate WorkingDirectory=__FINALPATH__/ # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory= -RuntimeDirectory=__APP__-celery +RuntimeDirectory=__APP__ -ExecStartPre=/bin/mkdir -p /var/log/__APP__-celery -ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__-celery +ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__ ExecStart=__FINALPATH__/venv/bin/celery multi start ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ diff --git a/conf/weblate.service b/conf/weblate.service index 9b3d597..16e6b43 100644 --- a/conf/weblate.service +++ b/conf/weblate.service @@ -12,14 +12,13 @@ PermissionsStartOnly=true # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory= RuntimeDirectory=__APP__ -ExecStartPre=/bin/mkdir -p /var/log/uwsgi/app/__APP__ -ExecStartPre=/bin/chown -R __APP__ /var/log/uwsgi/app/__APP__ +ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__ ExecStart=/usr/bin/uwsgi \ --ini /etc/uwsgi/apps-available/__APP__.ini \ --socket /var/run/__APP__/socket \ --chmod-socket=775 \ - --logto /var/log/uwsgi/app/__APP__/weblate \ + --logto /var/log/__APP__/weblate.log \ --processes 4 diff --git a/scripts/install b/scripts/install index 402c2f4..008a67a 100755 --- a/scripts/install +++ b/scripts/install @@ -306,7 +306,7 @@ chown -R "$app": "$final_path/avatar-cache" ynh_script_progression --message="Configuring log rotation..." --time --weight=1 # Use logrotate to manage application logfile(s) -ynh_use_logrotate --logfile "/var/log/$app-celery" +ynh_use_logrotate #================================================= # SETUP SSOWAT @@ -334,8 +334,8 @@ fi # ADVERTISE SERVICES IN ADMIN PANEL #================================================= -yunohost service add "$app" --log "/var/log/uwsgi/app/$app/weblate" -yunohost service add "$app-celery" --log "/var/log/$app-celery" +yunohost service add "$app" --log "/var/log/$app/weblate.log" +yunohost service add "$app-celery" --log "/var/log/$app/weblate-celery-w1.log" #================================================= # Start weblate diff --git a/scripts/restore b/scripts/restore index fe52137..895e5ba 100755 --- a/scripts/restore +++ b/scripts/restore @@ -108,8 +108,8 @@ systemctl enable "$app-celery" #================================================= # Add as a service -yunohost service add "$app" --log "/var/log/uwsgi/app/$app" -yunohost service add "$app-celery" --log "/var/log/$app-celery" +yunohost service add "$app" --log "/var/log/$app/weblate.log" +yunohost service add "$app-celery" --log "/var/log/$app/weblate-celery-w1.log" #================================================= # RESTORE THE CRON FILE diff --git a/scripts/upgrade b/scripts/upgrade index c5a9e9e..0fa5efa 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -165,6 +165,15 @@ then systemctl stop "$app-celery" fi +# (<3.8) log cleanups +if [[ -e "/var/log/uwsgi/app/$app" ]] +then + ynh_systemd_action --service_name="$app" --action="stop" + ynh_systemd_action --service_name="$app-celery" --action="stop" + ynh_secure_remove "/var/log/uwsgi/app/$app" + ynh_secure_remove "/var/log/$app-celery" +fi + #================================================= # CHECK THE PATH #================================================= @@ -403,7 +412,7 @@ ynh_replace_string "__FINALPATH__" "$final_path/" "/etc/cron.d/$app" ynh_script_progression --message="Upgrading logrotate configuration..." --time --weight=1 # Use logrotate to manage app-specific logfile(s) -ynh_use_logrotate --logfile "/var/log/$app-celery" --non-append +ynh_use_logrotate #================================================= # GENERIC FINALIZATION From dbd07e4d654f6ad38baded9a455bf02b0280760c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 28 Aug 2019 08:30:14 +0200 Subject: [PATCH 08/14] create log folder at restoration --- scripts/restore | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/restore b/scripts/restore index 895e5ba..2fa8132 100755 --- a/scripts/restore +++ b/scripts/restore @@ -128,6 +128,7 @@ ynh_restore_file "/usr/bin/hub" #================================================= ynh_restore_file --origin_path="/etc/logrotate.d/$app" +mkdir -p "/var/log/$app" #================================================= # GENERIC FINALIZATION From b7b857836e7e377c729a133814b09bbb31f3ce90 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 28 Aug 2019 08:30:33 +0200 Subject: [PATCH 09/14] fix nginx path traversal --- conf/nginx.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index f13dd41..329ccf7 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,3 +1,4 @@ +#sub_path_only rewrite ^__PATH__$ __PATH__/ permanent; # https://github.com/WeblateOrg/weblate/blob/master/examples/weblate.nginx.conf location ^__PATH__/favicon.ico$ { # DATA_DIR/static/favicon.ico @@ -23,7 +24,7 @@ location __PATH__/media/ { expires 30d; } -location __PATH__ { +location __PATH__/ { # Path to source alias __FINALPATH__/ ; @@ -34,4 +35,4 @@ location __PATH__ { # Include SSOWAT user panel. include conf.d/yunohost_panel.conf.inc; -} \ No newline at end of file +} From c1b8cef3fe65d1599a44894d34a78b52f355e316 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 28 Aug 2019 08:31:15 +0200 Subject: [PATCH 10/14] remove obsolete upgrade steps --- scripts/upgrade | 127 +----------------------------------------------- 1 file changed, 2 insertions(+), 125 deletions(-) diff --git a/scripts/upgrade b/scripts/upgrade index 0fa5efa..63dc8c8 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -75,96 +75,11 @@ elif [ "$is_public" = "No" ]; then is_public=0 fi -# (<2.17) -if [ -z "$db_name" ]; then # If db_name doesn't exist, create it - db_name=$(ynh_sanitize_dbid "$app") - ynh_app_setting_set "$app" db_name "$db_name" -fi - -# (<3.2) -migrate_to_python3=0 -if [ -e "$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" ]; then - settings="$final_path/venv/lib/python2.7/site-packages/weblate/settings.py" - migrate_to_python3=1 - cp "$settings" "$final_path/settings.py" -else - settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" -fi - -# (<2.17) save memc_port if it doesn't exist -if [[ -z "$memc_port" ]] -then - memc_port=$(cat "$settings" \ - | grep "'LOCATION': '127.0.0.1:" \ - | sed "s|.*:\\(.*\\)'.*|\\1|") - ynh_app_setting_set "$app" memc_port "$memc_port" -fi - -# Weblate requires an update to 3.0 before 3.1 -# the upgrade hook will launch this script again to make sure it works -if [[ -z "$migration311" && $previous_version = "2.20" ]] -then - # $migration311 is not set, version is <3.0 - migration311="two_steps_upgrade_3.0to3.1-needed" - - mig_type=$(cat "/etc/yunohost/apps/$YNH_APP_ID/status.json" | sed -e 's/.*"type":.*"\(.\+\)".*/\1/gi') - - if [ "$mig_type" = "file" ] - then - mig_parm=$(cat "/etc/yunohost/apps/$YNH_APP_ID/status.json" | sed -e 's/.*"path": "\(.\+\)",.*/\1/gi') - else - mig_parm=$(cat "/etc/yunohost/apps/$YNH_APP_ID/status.json" | sed -e 's/.*"url": "\(.\+\)", "type".*/\1/gi') - fi - - ynh_app_setting_set "$app" migration311 "$migration311" - ynh_app_setting_set "$app" migration311_type "$mig_type" - ynh_app_setting_set "$app" migration311_parm "$mig_parm" - current_version="3.0.1" -fi - -# (<3.2) -if [ -z "$redis_db" ]; then - redis_db=$(ynh_redis_get_free_db) - ynh_app_setting_set "$app" redis_db "$redis_db" -fi - -# (<2.20) remove complex uwsgi systemd template -if [ -e "/etc/systemd/system/uwsgi-app@.service" ] -then - systemctl stop "uwsgi-app@$app" - # if somehow the service doesn't exist, add it - yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" - yunohost service remove "uwsgi-app@$app.service" - ynh_secure_remove "/etc/systemd/system/uwsgi-app@.socket" - ynh_secure_remove "/etc/systemd/system/uwsgi-app@.service" - # remove old log file (used by the new service file) - ynh_secure_remove "/var/log/uwsgi/app/weblate" - systemctl daemon-reload -fi - -# (<2.18) move hub to the correct folder -if [ -e "$final_path/bin/hub" ] -then - mv "$final_path/bin/hub" /usr/bin/ - chown root:root /usr/bin/hub -fi - if [[ -d "$final_path/bin/" ]] then ynh_secure_remove "$final_path/bin/" fi -# Make sure the uwsgi service is stoped -if [[ -e "/var/run/$app/socket" ]] -then - systemctl stop "$app" -fi - -if [[ -e "/var/run/$app-celery/$app-w1.pid" ]] -then - systemctl stop "$app-celery" -fi - # (<3.8) log cleanups if [[ -e "/var/log/uwsgi/app/$app" ]] then @@ -255,15 +170,9 @@ ynh_store_file_checksum "$finaluwsgiini" # PIP INSTALLATION #================================================= -if [ "$migrate_to_python3" -eq 1 ] -then - ynh_script_progression --message="Rebuild the virtualenv with Python3..." --time --weight=1 - ynh_secure_remove "${final_path}/venv" - virtualenv --python=python3 "${final_path}/venv" -fi +ynh_script_progression --message="Install weblate using PIP..." --time --weight=10 ( - ynh_script_progression --message="Install weblate using PIP..." --time --weight=10 set +o nounset source "${final_path}/venv/bin/activate" set -o nounset @@ -276,17 +185,12 @@ fi pip install django_sendmail_backend ) -if [ "$migrate_to_python3" -eq 1 ] -then - mv "$final_path/settings.py" "$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" - settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" -fi - #================================================= # CONFIG FILE UPGRADE #================================================= ynh_script_progression --message="Create weblate configuration file..." --time --weight=1 # save old settings file +settings="$final_path/venv/lib/python3.5/site-packages/weblate/settings.py" old_settings="$final_path/settings.$previous_version.old.py" @@ -361,37 +265,10 @@ ynh_script_progression --message="Run migration scripts..." --time --weight=1 export DJANGO_SETTINGS_MODULE="weblate.settings" cd "${final_path}" - # https://docs.weblate.org/en/latest/admin/upgrade.html#upgrade-3 - # https://github.com/WeblateOrg/docker/blob/386aa8d98bb57dfec3707680827d4e4f4d79e3fd/start#L81-L88 - if [[ $current_version = "3.0.1" ]] - then - weblate showmigrations --plan > /tmp/migrations.txt - if grep -Fq '[X] auth.0001_initial' /tmp/migrations.txt && grep -Fq '[ ] weblate_auth.0001_initial' /tmp/migrations.txt ; then - ynh_replace_string "AUTH_USER_MODEL" "#AUTH_USER_MODEL" "$settings" - weblate migrate weblate_auth 0001 - ynh_replace_string "#AUTH_USER_MODEL" "AUTH_USER_MODEL" "$settings" - fi - - ynh_secure_remove /tmp/migrations.txt - fi - - weblate migrate --noinput weblate collectstatic --noinput weblate setuplang weblate setupgroups - - if [[ $previous_version = "2.16" ]] || \ - [[ $previous_version = "2.17.1" ]] || \ - [[ $previous_version = "2.18" ]] - then - weblate loadpo --all --lang dsb - weblate loadpo --all --lang he - weblate loadpo --all --lang hsb - weblate loadpo --all --lang kw - weblate loadpo --all --lang lt - weblate loadpo --all --lang lv - fi ) # Recalculate and store the config file checksum into the app settings From 7817df46ce35e5d7787666b0b5fed5d9c26afcb3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Wed, 28 Aug 2019 18:23:05 +0200 Subject: [PATCH 11/14] add ruamel.yaml with pip --- scripts/install | 2 +- scripts/upgrade | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install b/scripts/install index 008a67a..98bd281 100755 --- a/scripts/install +++ b/scripts/install @@ -189,7 +189,7 @@ virtualenv --python=python3 "${final_path}/venv" # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon ruamel.yaml # specific to YunoHost package: pip install django_sendmail_backend ) diff --git a/scripts/upgrade b/scripts/upgrade index 63dc8c8..cc50c78 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -180,7 +180,7 @@ ynh_script_progression --message="Install weblate using PIP..." --time --weight= # prevent error: "command: 'install_requires' must be a string or list of strings containing valid project/version requirement specifiers" pip install --upgrade setuptools pip install Weblate=="$current_version" - pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon + pip install pytz python-bidi PyYaML Babel pyuca pylibravatar py3dns psycopg2-binary phply django-redis hiredis aeidon ruamel.yaml # specific to YunoHost package: pip install django_sendmail_backend ) From fe753d3b365227f22a8ea7be2714a2e9b626cfe9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 30 Aug 2019 22:20:01 +0200 Subject: [PATCH 12/14] simplify nginx --- conf/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index 329ccf7..61637a1 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,12 +1,12 @@ #sub_path_only rewrite ^__PATH__$ __PATH__/ permanent; # https://github.com/WeblateOrg/weblate/blob/master/examples/weblate.nginx.conf -location ^__PATH__/favicon.ico$ { +location = __PATH__/favicon.ico { # DATA_DIR/static/favicon.ico alias __FINALPATH__/data/static/favicon.ico; expires 30d; } -location ^__PATH__/robots.txt$ { +location = __PATH__/robots.txt { # DATA_DIR/static/robots.txt alias __FINALPATH__/data/static/robots.txt; expires 30d; From f27261fe4e0568bd3f09e1f0d59f1b550556214c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 30 Aug 2019 22:21:03 +0200 Subject: [PATCH 13/14] Enable HSTS in Django --- conf/settings_history/settings.3.8.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/settings_history/settings.3.8.py b/conf/settings_history/settings.3.8.py index e2a22fc..19c6901 100644 --- a/conf/settings_history/settings.3.8.py +++ b/conf/settings_history/settings.3.8.py @@ -631,8 +631,8 @@ X_FRAME_OPTIONS = 'DENY' SECURE_CONTENT_TYPE_NOSNIFF = True # Optionally enable HSTS -SECURE_HSTS_SECONDS = 0 -SECURE_HSTS_PRELOAD = False +SECURE_HSTS_SECONDS = 63072000 # as in Strict-Transport-Security : max-age=yunohost/data/templates/nginx/yunohost_admin.conf +SECURE_HSTS_PRELOAD = False # already done by YunoHost SECURE_HSTS_INCLUDE_SUBDOMAINS = False # URL of login From 4214a1391c10b43abed0fb2679a25ac2d18b75cf Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Fri, 30 Aug 2019 22:21:30 +0200 Subject: [PATCH 14/14] align celery config with upstream --- conf/celery-weblate | 17 +++++++++-------- conf/celery-weblate.service | 12 ++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/conf/celery-weblate b/conf/celery-weblate index 7b2d738..1005d7c 100644 --- a/conf/celery-weblate +++ b/conf/celery-weblate @@ -1,18 +1,18 @@ # Name of nodes to start -# here we have a single node -CELERYD_NODES="w1" -# or we could have three nodes: -#CELERYD_NODES="w1 w2 w3" +CELERYD_NODES="celery notify search memory" + +# Absolute or relative path to the 'celery' command: +CELERY_BIN="__FINALPATH__/venv/bin/celery" # App instance to use # comment out this line if you don't use an app CELERY_APP="weblate" -# How to call manage.py -CELERYD_MULTI="multi" - # Extra command-line arguments to the worker -CELERYD_OPTS="--beat" +CELERYD_OPTS="--beat:celery --concurrency:celery=4 --queues:celery=celery --prefetch-multiplier:celery=4 \ +--concurrency:notify=4 --queues:notify=notify --prefetch-multiplier:notify=4 \ +--concurrency:search=1 --queues:search=search --prefetch-multiplier:search=2000 \ +--concurrency:memory=1 --queues:memory=memory --prefetch-multiplier:memory=2000" # - %n will be replaced with the first part of the nodename. # - %I will be replaced with the current child process index @@ -21,4 +21,5 @@ CELERYD_PID_FILE="/var/run/__APP__/weblate-celery-%n.pid" CELERYD_LOG_FILE="/var/log/__APP__/weblate-celery-%n%I.log" CELERYD_LOG_LEVEL="INFO" +# Internal Weblate variable to indicate we're running inside Celery CELERY_WORKER_RUNNING="1" diff --git a/conf/celery-weblate.service b/conf/celery-weblate.service index a6fe170..337cf35 100644 --- a/conf/celery-weblate.service +++ b/conf/celery-weblate.service @@ -14,14 +14,14 @@ RuntimeDirectory=__APP__ ExecStartPre=/bin/chown -R __APP__ /var/log/__APP__ -ExecStart=__FINALPATH__/venv/bin/celery multi start ${CELERYD_NODES} \ +ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} -ExecStop=__FINALPATH__/venv/bin/celery multi stopwait ${CELERYD_NODES} \ - --pidfile=${CELERYD_PID_FILE} -ExecReload=__FINALPATH__/venv/bin/celery multi restart ${CELERYD_NODES} \ + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' +ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} \ + --pidfile=${CELERYD_PID_FILE}' +ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} \ -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} \ - --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} + --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS}' [Install] WantedBy=multi-user.target