YunoHost-Apps/ffsync_ynh
1
0
Fork 0
mirror of https://github.com/YunoHost-Apps/ffsync_ynh.git synced 2024-09-03 18:26:38 +02:00
Code Issues Activity Actions

sources modified as a submodule.

Browse source
This commit is contained in:
Pierre-Antoine Rault 2017-03-01 16:36:43 +01:00
parent 6707f664bb
commit 1fef425bbd
15 changed files with 4 additions and 713 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored Normal file
View file

@ -0,0 +1,3 @@
[submodule "sources_orig"]
path = sources
url = https://github.com/mozilla-services/syncserver

1
sources Submodule

@ -0,0 +1 @@
Subproject commit 0b784329b2c3246d66ebb565e50f2e4f6ebb1c79

9
sources/.gitignore vendored
View file

@ -1,9 +0,0 @@
*.pyc
*.mako.py
local
*.egg-info
*.swp
\.coverage
*~
nosetests.xml
syncserver.db

20
sources/.travis.yml
View file

@ -1,20 +0,0 @@
language: python
python:
- "2.6"
- "2.7"
notifications:
email:
- rfkelly@mozilla.com
irc:
channels:
- "irc.mozilla.org#services-dev"
use_notice: false
skip_join: false
install:
- make build
script:
- make test

48
sources/Dockerfile
View file

@ -1,48 +0,0 @@
##########################################################
# /!\ WARNING /!\ #
# This is completely experimental. Use at your own risk. #
# Also, learn you some docker: #
# http://docker.io/gettingstarted #
##########################################################
FROM debian:7.4
MAINTAINER Dan Callahan <dan.callahan@gmail.com>
# Base system setup
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install --no-install-recommends -y \
vim locales \
&& apt-get clean
RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
ENV LANG C.UTF-8
RUN useradd --create-home app
# Build the Sync server
RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
libzmq-dev \
python-dev \
python-virtualenv \
&& apt-get clean
USER app
RUN mkdir -p /home/app/syncserver
ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/
ADD ./syncserver/ /home/app/syncserver/syncserver/
WORKDIR /home/app/syncserver
RUN make build
# Run the Sync server
EXPOSE 5000
ENTRYPOINT ["/usr/bin/make"]
CMD ["serve"]

3
sources/MANIFEST.in
View file

@ -1,3 +0,0 @@
include syncserver.ini
include syncserver.wsgi
include syncserver/tests.ini

50
sources/Makefile
View file

@ -1,50 +0,0 @@
SYSTEMPYTHON = `which python2 python | head -n 1`
VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON)
ENV = ./local
TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests)
# Hackety-hack around OSX system python bustage.
# The need for this should go away with a future osx/xcode update.
ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future
# Hackety-hack around errors duing compile of ultramemcached.
CFLAGS = "-Wno-error -Wno-error=format-security"
INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install
.PHONY: all
all: build
.PHONY: build
build: | $(ENV)/COMPLETE
$(ENV)/COMPLETE: requirements.txt
$(VIRTUALENV) --no-site-packages $(ENV)
$(INSTALL) -r requirements.txt
$(ENV)/bin/python ./setup.py develop
touch $(ENV)/COMPLETE
.PHONY: test
test: | $(TOOLS)
$(ENV)/bin/flake8 ./syncserver
$(ENV)/bin/nosetests -s syncstorage.tests
# Tokenserver tests currently broken due to incorrect file paths
# $(ENV)/bin/nosetests -s tokenserver.tests
# Test against a running server
$(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
sleep 2; \
$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
--use-token-server http://localhost:5000/token/1.0/sync/1.5; \
kill $$SERVER_PID
$(TOOLS): | $(ENV)/COMPLETE
$(INSTALL) nose flake8
.PHONY: serve
serve: | $(ENV)/COMPLETE
$(ENV)/bin/gunicorn --paste ./syncserver.ini
.PHONY: clean
clean:
rm -rf $(ENV)

73
sources/README.rst
View file

@ -1,73 +0,0 @@
Run-Your-Own Firefox Sync Server
================================
This is an all-in-one package for running a self-hosted Firefox Sync server.
If bundles the "tokenserver" project for authentication and the "syncstorage"
project for storage, produce a single stand-alone webapp.
Complete installation instructions are available at:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Quickstart
----------
The Sync Server software runs using **python 2.6** or later, and the build
process requires **make** and **virtualenv**. You will need to have the
following packages (or similar, depending on your operating system) installed:
- python2.7
- python2.7-dev
- python-virtualenv
- make
Take a checkout of this repository, then run "make build" to pull in the
necessary python package dependencies::
$ git clone https://github.com/mozilla-services/syncserver
$ cd syncserver
$ make build
To sanity-check that things got installed correctly, do the following::
$ make test
Now you can run the server::
$ make serve
This should start a server on http://localhost:5000/.
Now go into Firefox's `about:config` page, search for a setting named
"tokenServerURI", and change it to point to your server::
services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5
Firefox should now sync against your local server rather than the default
Mozilla-hosted servers.
For more details on setting up a stable deployment, see:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Customization
-------------
All customization of the server can be done by editing the file
"syncserver.ini", which contains lots of comments to help you on
your way. Things you might like to change include:
* The client-visible hostname for your server. Edit the "public_url"
key under the [syncstorage] section.
* The database in which to store sync data. Edit the "sqluri" setting
under the [syncstorage] section.
Questions, Feedback
-------------------
- IRC channel: #sync. See http://irc.mozilla.org/
- Mailing list: https://mail.mozilla.org/listinfo/services-dev

12
sources/requirements.txt
View file

@ -1,12 +0,0 @@
cornice==0.16.2
gunicorn==19.1.1
pyramid==1.5
requests==2.7
simplejson==3.4
SQLAlchemy==0.9.4
unittest2==0.5.1
zope.component==4.2.1
configparser==3.5.0b2
https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip

16
sources/setup.py
View file

@ -1,16 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup
entry_points = """
[paste.app_factory]
main = syncserver:main
"""
setup(
name='syncserver',
version="1.5.2",
packages=['syncserver'],
entry_points=entry_points
)

45
sources/syncserver.ini
View file

@ -1,45 +0,0 @@
[server:main]
use = egg:gunicorn
host = 0.0.0.0
port = 5000
workers = 1
timeout = 30
[app:main]
use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
# The following command will give a suitable value on *nix systems:
#
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
#secret = INSERT_SECRET_KEY_HERE
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
# allow_new_users = false
# Set this to "true" to work around a mismatch between public_url and
# the application URL as seen by python, which can happen in certain reverse-
# proxy hosting setups. It will overwrite the WSGI environ dict with the
# details from public_url. This could have security implications if e.g.
# you tell the app that it's on HTTPS but it's really on HTTP, so it should
# only be used as a last resort and after careful checking of server config.
force_wsgi_environ = false
# Uncomment and edit the following to use a local BrowserID verifier
# rather than posting assertions to the mozilla-hosted verifier.
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

40
sources/syncserver.wsgi
View file

@ -1,40 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import site
from logging.config import fileConfig
from ConfigParser import NoSectionError
# detecting if virtualenv was used in this dir
_CURDIR = os.path.dirname(os.path.abspath(__file__))
_PY_VER = sys.version.split()[0][:3]
_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages')
# adding virtualenv's site-package and ordering paths
saved = sys.path[:]
if os.path.exists(_SITE_PKG):
site.addsitedir(_SITE_PKG)
for path in sys.path:
if path not in saved:
saved.insert(0, path)
sys.path[:] = saved
# setting up the egg cache to a place where apache can write
os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs'
# setting up logging
ini_file = os.path.join(_CURDIR, 'syncserver.ini')
try:
fileConfig(ini_file)
except NoSectionError:
pass
# running the app using Paste
from paste.deploy import loadapp
application = loadapp('config:%s'% ini_file)

157
sources/syncserver/__init__.py
View file

@ -1,157 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import logging
from urlparse import urlparse, urlunparse
from pyramid.response import Response
from pyramid.events import NewRequest, subscriber
import mozsvc.config
from tokenserver.util import _JSONError
logger = logging.getLogger("syncserver")
def includeme(config):
"""Install SyncServer application into the given Pyramid configurator."""
# Set the umask so that files are created with secure permissions.
# Necessary for e.g. created-on-demand sqlite database files.
os.umask(0077)
# Sanity-check the deployment settings and provide sensible defaults.
settings = config.registry.settings
public_url = settings.get("syncserver.public_url")
if public_url is None:
raise RuntimeError("you much configure syncserver.public_url")
public_url = public_url.rstrip("/")
settings["syncserver.public_url"] = public_url
secret = settings.get("syncserver.secret")
if secret is None:
secret = os.urandom(32).encode("hex")
sqluri = settings.get("syncserver.sqluri")
if sqluri is None:
rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db")
# Configure app-specific defaults based on top-level configuration.
settings.pop("config", None)
if "tokenserver.backend" not in settings:
# Default to our simple static node-assignment backend
settings["tokenserver.backend"] =\
"syncserver.staticnode.StaticNodeAssignment"
settings["tokenserver.sqluri"] = sqluri
settings["tokenserver.node_url"] = public_url
settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}"
if "tokenserver.monkey_patch_gevent" not in settings:
# Default to no gevent monkey-patching
settings["tokenserver.monkey_patch_gevent"] = False
if "tokenserver.applications" not in settings:
# Default to just the sync-1.5 application
settings["tokenserver.applications"] = "sync-1.5"
if "tokenserver.secrets.backend" not in settings:
# Default to a single fixed signing secret
settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets"
settings["tokenserver.secrets.secrets"] = [secret]
if "tokenserver.allow_new_users" not in settings:
allow_new_users = settings.get("syncserver.allow_new_users")
if allow_new_users is not None:
settings["tokenserver.allow_new_users"] = allow_new_users
if "hawkauth.secrets.backend" not in settings:
# Default to the same secrets backend as the tokenserver
for key in settings.keys():
if key.startswith("tokenserver.secrets."):
newkey = "hawkauth" + key[len("tokenserver"):]
settings[newkey] = settings[key]
if "storage.backend" not in settings:
# Default to sql syncstorage backend
settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage"
settings["storage.sqluri"] = sqluri
settings["storage.create_tables"] = True
if "browserid.backend" not in settings:
# Default to remote verifier, with base of public_url as only audience
audience = urlunparse(urlparse(public_url)._replace(path=""))
settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
settings["browserid.audiences"] = audience
if "loggers" not in settings:
# Default to basic logging config.
root_logger = logging.getLogger("")
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO)
# Include the relevant sub-packages.
config.scan("syncserver")
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
# Add a top-level "it works!" view.
def itworks(request):
return Response("it works!")
config.add_route('itworks', '/')
config.add_view(itworks, route_name='itworks')
@subscriber(NewRequest)
def reconcile_wsgi_environ_with_public_url(event):
"""Event-listener that checks and tweaks WSGI environ based on public_url.
This is a simple trick to help ensure that the configured public_url
matches the actual deployed address. It fixes fixes parts of the WSGI
environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts
that seem obviously mis-configured (e.g. http:// versus https://).
It's very important to get public_url and WSGI environ matching exactly,
since they're used for browserid audience checking and HAWK signature
validation, so mismatches can easily cause strange and cryptic errors.
"""
request = event.request
public_url = request.registry.settings["syncserver.public_url"]
p_public_url = urlparse(public_url)
# If we don't have a SCRIPT_NAME, take it from the public_url.
# This is often the case if we're behind e.g. an nginx proxy that
# is serving us at some sub-path.
if not request.script_name:
request.script_name = p_public_url.path.rstrip("/")
# If the environ does not match public_url, requests are almost certainly
# going to fail due to auth errors. We can either bail out early, or we
# can forcibly clobber the WSGI environ with the values from public_url.
# This is a security risk if you've e.g. mis-configured the server, so
# it's not enabled by default.
application_url = request.application_url
if public_url != application_url:
if not request.registry.settings.get("syncserver.force_wsgi_environ"):
msg = "\n".join((
"The public_url setting doesn't match the application url.",
"This will almost certainly cause authentication failures!",
" public_url setting is: %s" % (public_url,),
" application url is: %s" % (application_url,),
"You can disable this check by setting the force_wsgi_environ",
"option in your config file, but do so at your own risk.",
))
logger.error(msg)
raise _JSONError([msg], status_code=500)
request.scheme = p_public_url.scheme
request.host = p_public_url.netloc
request.script_name = p_public_url.path.rstrip("/")
def get_configurator(global_config, **settings):
"""Load a SyncStorge configurator object from deployment settings."""
config = mozsvc.config.get_configurator(global_config, **settings)
config.begin()
try:
config.include(includeme)
finally:
config.end()
return config
def main(global_config, **settings):
"""Load a SyncStorage WSGI app from deployment settings."""
config = get_configurator(global_config, **settings)
return config.make_wsgi_app()

221
sources/syncserver/staticnode.py
View file

@ -1,221 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Simple node-assignment backend using a single, static node.
This is a greatly-simplified node-assignment backend. It keeps user records
in an SQL database, but does not attempt to do any node management. All users
are implicitly assigned to a single, static node.
XXX TODO: move this into the tokenserver repo.
"""
import time
import urlparse
from mozsvc.exceptions import BackendError
from sqlalchemy import Column, Integer, String, BigInteger, Index
from sqlalchemy import create_engine, Table, MetaData
from sqlalchemy.pool import QueuePool
from sqlalchemy.sql import text as sqltext
from sqlalchemy.exc import IntegrityError
from tokenserver.assignment import INodeAssignment
from zope.interface import implements
metadata = MetaData()
users = Table(
"users",
metadata,
Column("uid", Integer(), primary_key=True, autoincrement=True,
nullable=False),
Column("service", String(32), nullable=False),
Column("email", String(255), nullable=False),
Column("generation", BigInteger(), nullable=False),
Column("client_state", String(32), nullable=False),
Column("created_at", BigInteger(), nullable=False),
Column("replaced_at", BigInteger(), nullable=True),
Index('lookup_idx', 'email', 'service', 'created_at'),
Index('clientstate_idx', 'email', 'service', 'client_state', unique=True),
)
_GET_USER_RECORDS = sqltext("""\
select
uid, generation, client_state
from
users
where
email = :email
and
service = :service
order by
created_at desc, uid desc
limit
20
""")
_CREATE_USER_RECORD = sqltext("""\
insert into
users
(service, email, generation, client_state, created_at, replaced_at)
values
(:service, :email, :generation, :client_state, :timestamp, NULL)
""")
_UPDATE_GENERATION_NUMBER = sqltext("""\
update
users
set
generation = :generation
where
service = :service and email = :email and
generation < :generation and replaced_at is null
""")
_REPLACE_USER_RECORDS = sqltext("""\
update
users
set
replaced_at = :timestamp
where
service = :service and email = :email
and replaced_at is null and created_at < :timestamp
""")
def get_timestamp():
"""Get current timestamp in milliseconds."""
return int(time.time() * 1000)
class StaticNodeAssignment(object):
implements(INodeAssignment)
def __init__(self, sqluri, node_url, **kw):
self.sqluri = sqluri
self.node_url = node_url
self.driver = urlparse.urlparse(sqluri).scheme.lower()
sqlkw = {
"logging_name": "syncserver",
"connect_args": {},
"poolclass": QueuePool,
"pool_reset_on_return": True,
}
if self.driver == "sqlite":
# We must mark it as safe to share sqlite connections between
# threads. The pool will ensure there's on race conditions.
sqlkw["connect_args"]["check_same_thread"] = False
# If using a :memory: database, we must use a QueuePool of size
# 1 so that a single connection is shared by all threads.
if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
sqlkw["pool_size"] = 1
sqlkw["max_overflow"] = 0
if "mysql" in self.driver:
# Guard against the db closing idle conections.
sqlkw["pool_recycle"] = 3600
self._engine = create_engine(sqluri, **sqlkw)
users.create(self._engine, checkfirst=True)
def get_user(self, service, email):
params = {'service': service, 'email': email}
res = self._engine.execute(_GET_USER_RECORDS, **params)
try:
row = res.fetchone()
if row is None:
return None
# The first row is the most up-to-date user record.
user = {
'email': email,
'uid': row.uid,
'node': self.node_url,
'generation': row.generation,
'client_state': row.client_state,
'old_client_states': {}
}
# Any subsequent rows are due to old client-state values.
row = res.fetchone()
while row is not None:
user['old_client_states'][row.client_state] = True
row = res.fetchone()
return user
finally:
res.close()
def allocate_user(self, service, email, generation=0, client_state=''):
params = {
'service': service, 'email': email, 'generation': generation,
'client_state': client_state, 'timestamp': get_timestamp()
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
raise
return self.get_user(service, email)
else:
res.close()
return {
'email': email,
'uid': res.lastrowid,
'node': self.node_url,
'generation': generation,
'client_state': client_state,
'old_client_states': {}
}
def update_user(self, service, user, generation=None, client_state=None):
if client_state is None:
# uid can stay the same, just update the generation number.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation,
}
res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# reject previously-seen client-state strings.
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# need to create a new record for new client_state.
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'generation': generation, 'client_state': client_state,
'timestamp': now,
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
user.update(self.get_user(service, user['email']))
else:
self.get_user(service, user['email'])
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
res.close()
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
params = {
'service': service, 'email': user['email'], 'timestamp': now
}
res = self._engine.execute(_REPLACE_USER_RECORDS, **params)
res.close()

19
sources/syncserver/tests.ini
View file

@ -1,19 +0,0 @@
[server:main]
use = egg:gunicorn
host = 0.0.0.0
port = 5000
workers = 1
timeout = 30
[app:main]
use = egg:SyncServer
[syncserver]
# This must be edited to point to the public URL of your server.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
#secret = INSERT_SECRET_KEY_HERE