YunoHost-Apps/ffsync_ynh
1
0
Fork 0
mirror of https://github.com/YunoHost-Apps/ffsync_ynh.git synced 2024-09-03 18:26:38 +02:00
Code Issues Activity Actions

update ffsync 1.5

Browse source
This commit is contained in:
Adrien Beudin 2014-11-15 16:24:32 +00:00
parent 363f126eeb
commit 0d7e8fb9ae
19 changed files with 771 additions and 178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
conf/ffsync.ini Normal file
View file

@ -0,0 +1,12 @@
[uwsgi]
chmod-socket = 660
master = true
enable-threads = true
workers = 2
no-orphans = true
log-date = true
uid = ffsync
gid = ffsync
pythonpath = /opt/yunohost/ffsync/local/lib/python2.7/site-packages
wsgi-file = /opt/yunohost/ffsync/syncserver.wsgi
vacuum = true

84
conf/ffsync.sql
View file

@ -1,84 +0,0 @@
-- MySQL dump 10.13 Distrib 5.5.31, for debian-linux-gnu (x86_64)
--
-- Host: localhost Database: ffsync
-- ------------------------------------------------------
-- Server version 5.5.31-0+wheezy1
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `users`
--
DROP TABLE IF EXISTS `users`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `users` (
`username` varchar(255) NOT NULL DEFAULT '',
`md5` varchar(124) DEFAULT NULL,
PRIMARY KEY (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `users`
--
LOCK TABLES `users` WRITE;
/*!40000 ALTER TABLE `users` DISABLE KEYS */;
/*!40000 ALTER TABLE `users` ENABLE KEYS */;
UNLOCK TABLES;
--
-- Table structure for table `wbo`
--
DROP TABLE IF EXISTS `wbo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `wbo` (
`username` varchar(100) NOT NULL DEFAULT '',
`id` varchar(65) NOT NULL DEFAULT '',
`collection` varchar(100) NOT NULL DEFAULT '',
`parentid` varchar(65) DEFAULT NULL,
`predecessorid` int(11) DEFAULT NULL,
`modified` double DEFAULT NULL,
`sortindex` int(11) DEFAULT NULL,
`payload` text,
`payload_size` int(11) DEFAULT NULL,
`ttl` int(11) DEFAULT NULL,
PRIMARY KEY (`username`,`collection`,`id`),
KEY `parentindex` (`username`,`parentid`),
KEY `predecessorindex` (`username`,`predecessorid`),
KEY `modifiedindex` (`username`,`collection`,`modified`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping data for table `wbo`
--
LOCK TABLES `wbo` WRITE;
/*!40000 ALTER TABLE `wbo` DISABLE KEYS */;
/*!40000 ALTER TABLE `wbo` ENABLE KEYS */;
UNLOCK TABLES;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-- Dump completed on 2013-12-12 11:27:30

26
conf/nginx.conf
View file

@ -1,16 +1,14 @@
location PATHTOCHANGE {
alias ALIASTOCHANGE;
if ($scheme = http) {
rewrite ^ https://$server_name$request_uri? permanent;
}
index index.php;
try_files $uri $uri/ /index.php?$args;
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_index index.php;
include fastcgi_params;
fastcgi_param REMOTE_USER $remote_user;
fastcgi_param PATH_INFO $fastcgi_path_info;
}
if ($scheme = http) {
rewrite ^ https://$server_name$request_uri? permanent;
}
try_files $uri @ffsync;
}
location @ffsync {
uwsgi_pass unix:///run/uwsgi/app/ffsync/socket;
include uwsgi_params;
# Include SSOWAT user panel.
include conf.d/yunohost_panel.conf.inc;
}

26
conf/settings.php
View file

@ -1,26 +0,0 @@
<?php
// you can disable registration to the firefox sync server here,
// by setting ENABLE_REGISTER to false
//
define("ENABLE_REGISTER", true);
// firefox sync server url, this should end with a /
// e.g. https://YourDomain.de/Folder_und_ggf_/index.php/
//
define("FSYNCMS_ROOT", "https://URLFFSYNC/index.php/");
// Database connection credentials
//
define("SQLITE_FILE", "weave_db");
define("MYSQL_ENABLE", true);
define("MYSQL_HOST", "localhost");
define("MYSQL_DB", "yunobase");
define("MYSQL_USER", "yunouser");
define("MYSQL_PASSWORD", "yunopass");
// Use bcrypt instead of MD5 for password hashing
define("BCRYPT", true);
define("BCRYPT_ROUNDS", 12);
?>

35
conf/syncserver.ini Normal file
View file

@ -0,0 +1,35 @@
[server:main]
use = egg:Paste#http
host = 0.0.0.0
port = 5000
[app:main]
use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = https://ynhbaseurl/
# This defines the database in which to store all server data.
sqluri = sqlite:////opt/yunohost/ffsync/syncserver.db
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
# The following command will give a suitable value on *nix systems:
#
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
#secret = INSERT_SECRET_KEY_HERE
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
# allow_new_users = false
# Uncomment and edit the following to use a local BrowserID verifier
# rather than posing assertions to the mozilla-hosted verifier.
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

39
conf/weave_storage.diff
View file

@ -1,39 +0,0 @@
--- weave_storage.php 2013-12-16 13:44:24.252240725 +0000
+++ weave_storage.php.new 2013-12-16 13:43:15.480260767 +0000
@@ -720,6 +720,36 @@
function create_user($username, $password)
{
+ $mail = $auth_user;
+ $dn = "ou=users,dc=yunohost,dc=org";
+ $filter = "(&(objectclass=inetOrgPerson)(mail=".$mail."))";
+ $justthese = array("uid");
+ // connect to ldap server
+ $ldapconn = ldap_connect("localhost")
+ or die("Could not connect to LDAP server.");
+
+ ldap_set_option($ldapconn, LDAP_OPT_PROTOCOL_VERSION, 3);
+ ldap_set_option($ldapconn, LDAP_OPT_REFERRALS, 0);
+
+ if ($ldapconn) {
+ $ldapbind = @ldap_bind($ldapconn);
+ if (! $ldapbind) {
+ log_error("create_user:" . $exception->getMessage());
+ error_log("create_user:" . $exception->getMessage());
+ return 0;
+ }
+ else {
+ $sr = ldap_search($ldapconn, $dn, $filter, $justthese);
+ $info = ldap_get_entries($ldapconn, $sr);
+ if ( ! $info["count"]) {
+ log_error("create_user:" . $exception->getMessage());
+ error_log("create_user:" . $exception->getMessage());
+ return 0;
+ }
+ }
+ @ldap_unbind($ldapconn);
+ }
+
log_error("Create User - Username: ".$username."|".$password);
try

60
scripts/install
View file

@ -17,33 +17,65 @@ db_pwd=$(dd if=/dev/urandom bs=1 count=200 2> /dev/null | tr -c -d '[A-Za-z0-9]'
db_user=ffsync
# Initialize database and store mysql password for upgrade
sudo yunohost app initdb $db_user -p $db_pwd -s $(readlink -e ../conf/ffsync.sql)
sudo yunohost app initdb $db_user -p $db_pwd
sudo yunohost app setting ffsync mysqlpwd -v $db_pwd
# Check depends installation
sudo apt-get install python-dev git-core python-virtualenv uwsgi
# Check Swap
tmp_swap_file=/tmp/ffsync_swapfile
if [ $(sudo swapon -s | wc -l) = 1 ];
then
sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
sudo chmod 600 $tmp_swap_file
sudo mkswap $tmp_swap_file
sudo swapon $tmp_swap_file
fi
# Copy files to the right place
final_path=/var/www/ffsync
final_path=/opt/yunohost/ffsync
sudo mkdir -p $final_path
sudo cp -a ../sources/* $final_path
sudo cp ../conf/settings.php $final_path/
#sudo cp ../conf/settings.php $final_path/
# Change variables in FSyncMS configuration
sudo sed -i "s/yunouser/$db_user/g" $final_path/settings.php
sudo sed -i "s/yunopass/$db_pwd/g" $final_path/settings.php
sudo sed -i "s/yunobase/$db_user/g" $final_path/settings.php
sudo sed -i "s@URLFFSYNC@$domain$path@g" $final_path/settings.php
#sudo sed -i "s/yunouser/$db_user/g" $final_path/settings.php
#sudo sed -i "s/yunopass/$db_pwd/g" $final_path/settings.php
#sudo sed -i "s/yunobase/$db_user/g" $final_path/settings.php
#sudo sed -i "s@URLFFSYNC@$domain$path@g" $final_path/settings.php
# Set permissions to roundcube directory
sudo chown -R www-data: $final_path
# Set permissions to ffsync directory
sudo useradd ffsync -d $final_path
sudo chown ffsync:ffsync -R $final_path
# Copy uwsgi config
sudo cp ../conf/ffsync.ini /etc/uwsgi/apps-available/
sudo ln -s /etc/uwsgi/apps-available/ffsync.ini /etc/uwsgi/apps-enabled/
# Modify Nginx configuration file and copy it to Nginx conf directory
sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf*
sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf*
sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf
sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
sudo cp ../conf/weave_storage.diff /tmp/weave_storage.diff
#sudo patch -p0 $final_path/weave_storage.php < /tmp/weave_storage.diff
sudo rm $final_path/setup.php
sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini
# Init virtualenv
cd $final_path && sudo make build
# Disable swapfile
if [ -f $tmp_swap_file ];
then
sudo swapoff $tmp_swap_file
sudo rm -f $tmp_swap_file
fi
# Fix permission
sudo find $final_path/ -type d -exec chmod 2755 {} \;
sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
sudo usermod -a -G ffsync www-data
# Reload Nginx and regenerate SSOwat conf
sudo service uwsgi restart
sudo service nginx reload
sudo yunohost app setting ffsync skipped_uris -v "/"
sudo yunohost app ssowatconf

9
scripts/remove
View file

@ -6,5 +6,12 @@ root_pwd=$(sudo cat /etc/yunohost/mysql)
domain=$(sudo yunohost app setting ffsync domain)
mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;"
sudo rm -rf /var/www/ffsync
sudo rm -rf /opt/yunohost/ffsync
sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf
sudo rm -f /etc/uwsgi/apps-enabled/searx.ini
sudo rm -f /etc/uwsgi/apps-available/searx.ini
sudo service uwsgi stop
sudo killall uwsgi
sudo service uwsgi start
sudo userdel ffsync

47
sources/Dockerfile Normal file
View file

@ -0,0 +1,47 @@
##########################################################
# /!\ WARNING /!\ #
# This is completely experimental. Use at your own risk. #
# Also, learn you some docker: #
# http://docker.io/gettingstarted #
##########################################################
FROM debian:7.4
MAINTAINER Dan Callahan <dan.callahan@gmail.com>
# Base system setup
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install --no-install-recommends -y \
vim locales \
&& apt-get clean
RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
ENV LANG C.UTF-8
RUN useradd --create-home app
# Build the Sync server
RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
libzmq-dev \
python-dev \
python-virtualenv \
&& apt-get clean
USER app
RUN mkdir -p /home/app/syncserver
ADD ./ /home/app/syncserver
WORKDIR /home/app/syncserver
RUN make build
# Run the Sync server
EXPOSE 5000
ENTRYPOINT ["/usr/bin/make"]
CMD ["serve"]

3
sources/MANIFEST.in Normal file
View file

@ -0,0 +1,3 @@
include syncserver.ini
include syncserver.wsgi
include syncserver/tests.ini

50
sources/Makefile Normal file
View file

@ -0,0 +1,50 @@
SYSTEMPYTHON = `which python2 python | head -n 1`
VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON)
ENV = ./local
TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests)
# Hackety-hack around OSX system python bustage.
# The need for this should go away with a future osx/xcode update.
ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future
# Hackety-hack around errors duing compile of ultramemcached.
CFLAGS = -Wno-error
INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install
.PHONY: all
all: build
.PHONY: build
build: | $(ENV)/COMPLETE
$(ENV)/COMPLETE: requirements.txt
$(VIRTUALENV) --no-site-packages $(ENV)
$(INSTALL) -r requirements.txt
$(ENV)/bin/python ./setup.py develop
touch $(ENV)/COMPLETE
.PHONY: test
test: | $(TOOLS)
$(ENV)/bin/flake8 ./syncserver
$(ENV)/bin/nosetests -s syncstorage.tests
# Tokenserver tests currently broken due to incorrect file paths
# $(ENV)/bin/nosetests -s tokenserver.tests
# Test against a running server
$(ENV)/bin/pserve syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
sleep 2; \
$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
--use-token-server http://localhost:5000/token/1.0/sync/1.5; \
kill $$SERVER_PID
$(TOOLS): | $(ENV)/COMPLETE
$(INSTALL) nose flake8
.PHONY: serve
serve: | $(ENV)/COMPLETE
$(ENV)/bin/pserve ./syncserver.ini
.PHONY: clean
clean:
rm -rf $(ENV)

73
sources/README.rst Normal file
View file

@ -0,0 +1,73 @@
Run-Your-Own Firefox Sync Server
================================
This is an all-in-one package for running a self-hosted Firefox Sync server.
If bundles the "tokenserver" project for authentication and the "syncstorage"
project for storage, produce a single stand-alone webapp.
Complete installation instructions are available at:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Quickstart
----------
The Sync Server software runs using **python 2.6** or later, and the build
process requires **make** and **virtualenv**. You will need to have the
following packages (or similar, depending on your operating system) installed:
- python2.7
- python2.7-dev
- python-virtualenv
- make
Take a checkout of this repository, then run "make build" to pull in the
necessary python package dependencies::
$ git clone https://github.com/mozilla-services/syncserver
$ cd syncserver
$ make build
To sanity-check that things got installed correctly, do the following::
$ make test
Now you can run the server::
$ make serve
This should start a server on http://localhost:5000/.
Now go into Firefox's `about:config` page, search for a setting named
"tokenServerURI", and change it to point to your server::
services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5
Firefox should now sync against your local server rather than the default
Mozilla-hosted servers.
For more details on setting up a stable deployment, see:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Customization
-------------
All customization of the server can be done by editing the file
"syncserver.ini", which contains lots of comments to help you on
your way. Things you might like to change include:
* The client-visible hostname for your server. Edit the "public_url"
key under the [syncstorage] section.
* The database in which to store sync data. Edit the "sqluri" setting
under the [syncstorage] section.
Questions, Feedback
-------------------
- IRC channel: #sync. See http://irc.mozilla.org/
- Mailing list: https://mail.mozilla.org/listinfo/services-dev

13
sources/requirements.txt Normal file
View file

@ -0,0 +1,13 @@
cornice==0.16.2
gunicorn==19.1.1
pyramid==1.5
requests==2.2.1
simplejson==3.4
SQLAlchemy==0.9.4
unittest2==0.5.1
https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip
# Newer releases of configparser have b/w compat bug:
# https://github.com/mozilla-services/syncserver/issues/39
configparser==3.3.0r2

16
sources/setup.py Normal file
View file

@ -0,0 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup
entry_points = """
[paste.app_factory]
main = syncserver:main
"""
setup(
name='syncserver',
version="1.5.1",
packages=['syncserver'],
entry_points=entry_points
)

35
sources/syncserver.ini Normal file
View file

@ -0,0 +1,35 @@
[server:main]
use = egg:Paste#http
host = 0.0.0.0
port = 5000
[app:main]
use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
# The following command will give a suitable value on *nix systems:
#
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
#secret = INSERT_SECRET_KEY_HERE
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
# allow_new_users = false
# Uncomment and edit the following to use a local BrowserID verifier
# rather than posing assertions to the mozilla-hosted verifier.
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

40
sources/syncserver.wsgi Normal file
View file

@ -0,0 +1,40 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import site
from logging.config import fileConfig
from ConfigParser import NoSectionError
# detecting if virtualenv was used in this dir
_CURDIR = os.path.dirname(os.path.abspath(__file__))
_PY_VER = sys.version.split()[0][:3]
_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages')
# adding virtualenv's site-package and ordering paths
saved = sys.path[:]
if os.path.exists(_SITE_PKG):
site.addsitedir(_SITE_PKG)
for path in sys.path:
if path not in saved:
saved.insert(0, path)
sys.path[:] = saved
# setting up the egg cache to a place where apache can write
os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs'
# setting up logging
ini_file = os.path.join(_CURDIR, 'syncserver.ini')
try:
fileConfig(ini_file)
except NoSectionError:
pass
# running the app using Paste
from paste.deploy import loadapp
application = loadapp('config:%s'% ini_file)

146
sources/syncserver/__init__.py Normal file
View file

@ -0,0 +1,146 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import logging
from urlparse import urlparse, urlunparse
from pyramid.response import Response
from pyramid.events import NewRequest, subscriber
import mozsvc.config
from tokenserver.util import _JSONError
logger = logging.getLogger("syncserver")
def includeme(config):
"""Install SyncServer application into the given Pyramid configurator."""
# Set the umask so that files are created with secure permissions.
# Necessary for e.g. created-on-demand sqlite database files.
os.umask(0077)
# Sanity-check the deployment settings and provide sensible defaults.
settings = config.registry.settings
public_url = settings.get("syncserver.public_url")
if public_url is None:
raise RuntimeError("you much configure syncserver.public_url")
public_url = public_url.rstrip("/")
settings["syncserver.public_url"] = public_url
secret = settings.get("syncserver.secret")
if secret is None:
secret = os.urandom(32).encode("hex")
sqluri = settings.get("syncserver.sqluri")
if sqluri is None:
rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db")
# Configure app-specific defaults based on top-level configuration.
settings.pop("config", None)
if "tokenserver.backend" not in settings:
# Default to our simple static node-assignment backend
settings["tokenserver.backend"] =\
"syncserver.staticnode.StaticNodeAssignment"
settings["tokenserver.sqluri"] = sqluri
settings["tokenserver.node_url"] = public_url
settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}"
if "tokenserver.monkey_patch_gevent" not in settings:
# Default to no gevent monkey-patching
settings["tokenserver.monkey_patch_gevent"] = False
if "tokenserver.applications" not in settings:
# Default to just the sync-1.5 application
settings["tokenserver.applications"] = "sync-1.5"
if "tokenserver.secrets.backend" not in settings:
# Default to a single fixed signing secret
settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets"
settings["tokenserver.secrets.secrets"] = [secret]
if "tokenserver.allow_new_users" not in settings:
allow_new_users = settings.get("syncserver.allow_new_users")
if allow_new_users is not None:
settings["tokenserver.allow_new_users"] = allow_new_users
if "hawkauth.secrets.backend" not in settings:
# Default to the same secrets backend as the tokenserver
for key in settings.keys():
if key.startswith("tokenserver.secrets."):
newkey = "hawkauth" + key[len("tokenserver"):]
settings[newkey] = settings[key]
if "storage.backend" not in settings:
# Default to sql syncstorage backend
settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage"
settings["storage.sqluri"] = sqluri
settings["storage.create_tables"] = True
if "browserid.backend" not in settings:
# Default to remote verifier, with base of public_url as only audience
audience = urlunparse(urlparse(public_url)._replace(path=""))
settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
settings["browserid.audiences"] = audience
if "loggers" not in settings:
# Default to basic logging config.
root_logger = logging.getLogger("")
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO)
# Include the relevant sub-packages.
config.scan("syncserver")
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
# Add a top-level "it works!" view.
def itworks(request):
return Response("it works!")
config.add_route('itworks', '/')
config.add_view(itworks, route_name='itworks')
@subscriber(NewRequest)
def reconcile_wsgi_environ_with_public_url(event):
"""Event-listener that checks and tweaks WSGI environ based on public_url.
This is a simple trick to help ensure that the configured public_url
matches the actual deployed address. It fixes fixes parts of the WSGI
environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts
that seem obviously mis-configured (e.g. http:// versus https://).
It's very important to get public_url and WSGI environ matching exactly,
since they're used for browserid audience checking and HAWK signature
validation, so mismatches can easily cause strange and cryptic errors.
"""
request = event.request
public_url = request.registry.settings["syncserver.public_url"]
p_public_url = urlparse(public_url)
# If we don't have a SCRIPT_NAME, take it from the public_url.
# This is often the case if we're behind e.g. an nginx proxy that
# is serving us at some sub-path.
if not request.script_name:
request.script_name = p_public_url.path.rstrip("/")
# Log a noisy error if the application url is different to what we'd
# expect based on public_url setting.
application_url = request.application_url
if public_url != application_url:
msg = "The public_url setting does not match the application url.\n"
msg += "This will almost certainly cause authentication failures!\n"
msg += " public_url setting is: %s\n" % (public_url,)
msg += " application url is: %s\n" % (application_url,)
logger.error(msg)
raise _JSONError([msg], status_code=500)
def get_configurator(global_config, **settings):
"""Load a SyncStorge configurator object from deployment settings."""
config = mozsvc.config.get_configurator(global_config, **settings)
config.begin()
try:
config.include(includeme)
finally:
config.end()
return config
def main(global_config, **settings):
"""Load a SyncStorage WSGI app from deployment settings."""
config = get_configurator(global_config, **settings)
return config.make_wsgi_app()

218
sources/syncserver/staticnode.py Normal file
View file

@ -0,0 +1,218 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Simple node-assignment backend using a single, static node.
This is a greatly-simplified node-assignment backend. It keeps user records
in an SQL database, but does not attempt to do any node management. All users
are implicitly assigned to a single, static node.
XXX TODO: move this into the tokenserver repo.
"""
import time
import urlparse
from mozsvc.exceptions import BackendError
from sqlalchemy import Column, Integer, String, BigInteger, Index
from sqlalchemy import create_engine, Table, MetaData
from sqlalchemy.pool import QueuePool
from sqlalchemy.sql import text as sqltext
from sqlalchemy.exc import IntegrityError
from tokenserver.assignment import INodeAssignment
from zope.interface import implements
metadata = MetaData()
users = Table(
"users",
metadata,
Column("uid", Integer(), primary_key=True, autoincrement=True,
nullable=False),
Column("service", String(32), nullable=False),
Column("email", String(255), nullable=False),
Column("generation", BigInteger(), nullable=False),
Column("client_state", String(32), nullable=False),
Column("created_at", BigInteger(), nullable=False),
Column("replaced_at", BigInteger(), nullable=True),
Index('lookup_idx', 'email', 'service', 'created_at'),
Index('clientstate_idx', 'email', 'service', 'client_state', unique=True),
)
_GET_USER_RECORDS = sqltext("""\
select
uid, generation, client_state
from
users
where
email = :email
and
service = :service
order by
created_at desc, uid desc
limit
20
""")
_CREATE_USER_RECORD = sqltext("""\
insert into
users
(service, email, generation, client_state, created_at, replaced_at)
values
(:service, :email, :generation, :client_state, :timestamp, NULL)
""")
_UPDATE_GENERATION_NUMBER = sqltext("""\
update
users
set
generation = :generation
where
service = :service and email = :email and
generation < :generation and replaced_at is null
""")
_REPLACE_USER_RECORDS = sqltext("""\
update
users
set
replaced_at = :timestamp
where
service = :service and email = :email
and replaced_at is null and created_at < :timestamp
""")
def get_timestamp():
"""Get current timestamp in milliseconds."""
return int(time.time() * 1000)
class StaticNodeAssignment(object):
implements(INodeAssignment)
def __init__(self, sqluri, node_url, **kw):
self.sqluri = sqluri
self.node_url = node_url
self.driver = urlparse.urlparse(sqluri).scheme.lower()
sqlkw = {
"logging_name": "syncserver",
"connect_args": {},
"poolclass": QueuePool,
"pool_reset_on_return": True,
}
if self.driver == "sqlite":
# We must mark it as safe to share sqlite connections between
# threads. The pool will ensure there's on race conditions.
sqlkw["connect_args"]["check_same_thread"] = False
# If using a :memory: database, we must use a QueuePool of size
# 1 so that a single connection is shared by all threads.
if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
sqlkw["pool_size"] = 1
sqlkw["max_overflow"] = 0
self._engine = create_engine(sqluri, **sqlkw)
users.create(self._engine, checkfirst=True)
def get_user(self, service, email):
params = {'service': service, 'email': email}
res = self._engine.execute(_GET_USER_RECORDS, **params)
try:
row = res.fetchone()
if row is None:
return None
# The first row is the most up-to-date user record.
user = {
'email': email,
'uid': row.uid,
'node': self.node_url,
'generation': row.generation,
'client_state': row.client_state,
'old_client_states': {}
}
# Any subsequent rows are due to old client-state values.
row = res.fetchone()
while row is not None:
user['old_client_states'][row.client_state] = True
row = res.fetchone()
return user
finally:
res.close()
def allocate_user(self, service, email, generation=0, client_state=''):
params = {
'service': service, 'email': email, 'generation': generation,
'client_state': client_state, 'timestamp': get_timestamp()
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
raise
return self.get_user(service, email)
else:
res.close()
return {
'email': email,
'uid': res.lastrowid,
'node': self.node_url,
'generation': generation,
'client_state': client_state,
'old_client_states': {}
}
def update_user(self, service, user, generation=None, client_state=None):
if client_state is None:
# uid can stay the same, just update the generation number.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation,
}
res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# reject previously-seen client-state strings.
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# need to create a new record for new client_state.
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'generation': generation, 'client_state': client_state,
'timestamp': now,
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
user.update(self.get_user(service, user['email']))
else:
self.get_user(service, user['email'])
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
res.close()
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
params = {
'service': service, 'email': user['email'], 'timestamp': now
}
res = self._engine.execute(_REPLACE_USER_RECORDS, **params)
res.close()

17
sources/syncserver/tests.ini Normal file
View file

@ -0,0 +1,17 @@
[server:main]
use = egg:Paste#http
host = 0.0.0.0
port = 5000
[app:main]
use = egg:SyncServer
[syncserver]
# This must be edited to point to the public URL of your server.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
#secret = INSERT_SECRET_KEY_HERE