diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5fc8fef --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sources_orig"] + path = sources + url = https://github.com/mozilla-services/syncserver diff --git a/sources b/sources new file mode 160000 index 0000000..0b78432 --- /dev/null +++ b/sources @@ -0,0 +1 @@ +Subproject commit 0b784329b2c3246d66ebb565e50f2e4f6ebb1c79 diff --git a/sources/.gitignore b/sources/.gitignore deleted file mode 100644 index 45b872e..0000000 --- a/sources/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.pyc -*.mako.py -local -*.egg-info -*.swp -\.coverage -*~ -nosetests.xml -syncserver.db diff --git a/sources/.travis.yml b/sources/.travis.yml deleted file mode 100644 index 2e45ae4..0000000 --- a/sources/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -language: python - -python: - - "2.6" - - "2.7" - -notifications: - email: - - rfkelly@mozilla.com - irc: - channels: - - "irc.mozilla.org#services-dev" - use_notice: false - skip_join: false - -install: - - make build - -script: - - make test diff --git a/sources/Dockerfile b/sources/Dockerfile deleted file mode 100644 index b66c21f..0000000 --- a/sources/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -########################################################## -# /!\ WARNING /!\ # -# This is completely experimental. Use at your own risk. # -# Also, learn you some docker: # -# http://docker.io/gettingstarted # -########################################################## - -FROM debian:7.4 -MAINTAINER Dan Callahan - -# Base system setup - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install --no-install-recommends -y \ - vim locales \ - && apt-get clean - -RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale - -ENV LANG C.UTF-8 - -RUN useradd --create-home app - -# Build the Sync server - -RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - build-essential \ - libzmq-dev \ - python-dev \ - python-virtualenv \ - && apt-get clean - -USER app - -RUN mkdir -p /home/app/syncserver -ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/ -ADD ./syncserver/ /home/app/syncserver/syncserver/ -WORKDIR /home/app/syncserver - -RUN make build - -# Run the Sync server - -EXPOSE 5000 - -ENTRYPOINT ["/usr/bin/make"] -CMD ["serve"] diff --git a/sources/MANIFEST.in b/sources/MANIFEST.in deleted file mode 100644 index 3ffd5cf..0000000 --- a/sources/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include syncserver.ini -include syncserver.wsgi -include syncserver/tests.ini diff --git a/sources/Makefile b/sources/Makefile deleted file mode 100644 index 2efa163..0000000 --- a/sources/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -SYSTEMPYTHON = `which python2 python | head -n 1` -VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON) -ENV = ./local -TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests) - -# Hackety-hack around OSX system python bustage. -# The need for this should go away with a future osx/xcode update. -ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future - -# Hackety-hack around errors duing compile of ultramemcached. -CFLAGS = "-Wno-error -Wno-error=format-security" - -INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install - - -.PHONY: all -all: build - -.PHONY: build -build: | $(ENV)/COMPLETE -$(ENV)/COMPLETE: requirements.txt - $(VIRTUALENV) --no-site-packages $(ENV) - $(INSTALL) -r requirements.txt - $(ENV)/bin/python ./setup.py develop - touch $(ENV)/COMPLETE - -.PHONY: test -test: | $(TOOLS) - $(ENV)/bin/flake8 ./syncserver - $(ENV)/bin/nosetests -s syncstorage.tests - # Tokenserver tests currently broken due to incorrect file paths - # $(ENV)/bin/nosetests -s tokenserver.tests - - # Test against a running server - $(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \ - sleep 2; \ - $(ENV)/bin/python -m syncstorage.tests.functional.test_storage \ - --use-token-server http://localhost:5000/token/1.0/sync/1.5; \ - kill $$SERVER_PID - -$(TOOLS): | $(ENV)/COMPLETE - $(INSTALL) nose flake8 - -.PHONY: serve -serve: | $(ENV)/COMPLETE - $(ENV)/bin/gunicorn --paste ./syncserver.ini - -.PHONY: clean -clean: - rm -rf $(ENV) diff --git a/sources/README.rst b/sources/README.rst deleted file mode 100644 index 3872786..0000000 --- a/sources/README.rst +++ /dev/null @@ -1,73 +0,0 @@ -Run-Your-Own Firefox Sync Server -================================ - -This is an all-in-one package for running a self-hosted Firefox Sync server. -If bundles the "tokenserver" project for authentication and the "syncstorage" -project for storage, produce a single stand-alone webapp. - -Complete installation instructions are available at: - - https://docs.services.mozilla.com/howtos/run-sync-1.5.html - - -Quickstart ----------- - -The Sync Server software runs using **python 2.6** or later, and the build -process requires **make** and **virtualenv**. You will need to have the -following packages (or similar, depending on your operating system) installed: - -- python2.7 -- python2.7-dev -- python-virtualenv -- make - -Take a checkout of this repository, then run "make build" to pull in the -necessary python package dependencies:: - - $ git clone https://github.com/mozilla-services/syncserver - $ cd syncserver - $ make build - -To sanity-check that things got installed correctly, do the following:: - - $ make test - -Now you can run the server:: - - $ make serve - -This should start a server on http://localhost:5000/. - -Now go into Firefox's `about:config` page, search for a setting named -"tokenServerURI", and change it to point to your server:: - - services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5 - -Firefox should now sync against your local server rather than the default -Mozilla-hosted servers. - -For more details on setting up a stable deployment, see: - - https://docs.services.mozilla.com/howtos/run-sync-1.5.html - - -Customization -------------- - -All customization of the server can be done by editing the file -"syncserver.ini", which contains lots of comments to help you on -your way. Things you might like to change include: - - * The client-visible hostname for your server. Edit the "public_url" - key under the [syncstorage] section. - - * The database in which to store sync data. Edit the "sqluri" setting - under the [syncstorage] section. - - -Questions, Feedback -------------------- - -- IRC channel: #sync. See http://irc.mozilla.org/ -- Mailing list: https://mail.mozilla.org/listinfo/services-dev diff --git a/sources/requirements.txt b/sources/requirements.txt deleted file mode 100644 index 2980219..0000000 --- a/sources/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -cornice==0.16.2 -gunicorn==19.1.1 -pyramid==1.5 -requests==2.7 -simplejson==3.4 -SQLAlchemy==0.9.4 -unittest2==0.5.1 -zope.component==4.2.1 -configparser==3.5.0b2 -https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip -https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip -http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip diff --git a/sources/setup.py b/sources/setup.py deleted file mode 100644 index 8340912..0000000 --- a/sources/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -from setuptools import setup - -entry_points = """ -[paste.app_factory] -main = syncserver:main -""" - -setup( - name='syncserver', - version="1.5.2", - packages=['syncserver'], - entry_points=entry_points -) diff --git a/sources/syncserver.ini b/sources/syncserver.ini deleted file mode 100644 index 7672182..0000000 --- a/sources/syncserver.ini +++ /dev/null @@ -1,45 +0,0 @@ -[server:main] -use = egg:gunicorn -host = 0.0.0.0 -port = 5000 -workers = 1 -timeout = 30 - -[app:main] -use = egg:syncserver - -[syncserver] -# This must be edited to point to the public URL of your server, -# i.e. the URL as seen by Firefox. -public_url = http://localhost:5000/ - -# This defines the database in which to store all server data. -#sqluri = sqlite:////tmp/syncserver.db - -# This is a secret key used for signing authentication tokens. -# It should be long and randomly-generated. -# The following command will give a suitable value on *nix systems: -# -# head -c 20 /dev/urandom | sha1sum -# -# If not specified then the server will generate a temporary one at startup. -#secret = INSERT_SECRET_KEY_HERE - -# Set this to "false" to disable new-user signups on the server. -# Only request by existing accounts will be honoured. -# allow_new_users = false - -# Set this to "true" to work around a mismatch between public_url and -# the application URL as seen by python, which can happen in certain reverse- -# proxy hosting setups. It will overwrite the WSGI environ dict with the -# details from public_url. This could have security implications if e.g. -# you tell the app that it's on HTTPS but it's really on HTTP, so it should -# only be used as a last resort and after careful checking of server config. -force_wsgi_environ = false - -# Uncomment and edit the following to use a local BrowserID verifier -# rather than posting assertions to the mozilla-hosted verifier. -# Audiences should be set to your public_url without a trailing slash. -#[browserid] -#backend = tokenserver.verifiers.LocalVerifier -#audiences = https://localhost:5000 diff --git a/sources/syncserver.wsgi b/sources/syncserver.wsgi deleted file mode 100644 index 4b0503c..0000000 --- a/sources/syncserver.wsgi +++ /dev/null @@ -1,40 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import sys -import site -from logging.config import fileConfig -from ConfigParser import NoSectionError - -# detecting if virtualenv was used in this dir -_CURDIR = os.path.dirname(os.path.abspath(__file__)) -_PY_VER = sys.version.split()[0][:3] -_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages') - -# adding virtualenv's site-package and ordering paths -saved = sys.path[:] - -if os.path.exists(_SITE_PKG): - site.addsitedir(_SITE_PKG) - -for path in sys.path: - if path not in saved: - saved.insert(0, path) - -sys.path[:] = saved - -# setting up the egg cache to a place where apache can write -os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs' - -# setting up logging -ini_file = os.path.join(_CURDIR, 'syncserver.ini') -try: - fileConfig(ini_file) -except NoSectionError: - pass - -# running the app using Paste -from paste.deploy import loadapp -application = loadapp('config:%s'% ini_file) diff --git a/sources/syncserver/__init__.py b/sources/syncserver/__init__.py deleted file mode 100644 index 696a430..0000000 --- a/sources/syncserver/__init__.py +++ /dev/null @@ -1,157 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import logging -from urlparse import urlparse, urlunparse - -from pyramid.response import Response -from pyramid.events import NewRequest, subscriber - -import mozsvc.config - -from tokenserver.util import _JSONError - -logger = logging.getLogger("syncserver") - - -def includeme(config): - """Install SyncServer application into the given Pyramid configurator.""" - # Set the umask so that files are created with secure permissions. - # Necessary for e.g. created-on-demand sqlite database files. - os.umask(0077) - - # Sanity-check the deployment settings and provide sensible defaults. - settings = config.registry.settings - public_url = settings.get("syncserver.public_url") - if public_url is None: - raise RuntimeError("you much configure syncserver.public_url") - public_url = public_url.rstrip("/") - settings["syncserver.public_url"] = public_url - - secret = settings.get("syncserver.secret") - if secret is None: - secret = os.urandom(32).encode("hex") - sqluri = settings.get("syncserver.sqluri") - if sqluri is None: - rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db") - - # Configure app-specific defaults based on top-level configuration. - settings.pop("config", None) - if "tokenserver.backend" not in settings: - # Default to our simple static node-assignment backend - settings["tokenserver.backend"] =\ - "syncserver.staticnode.StaticNodeAssignment" - settings["tokenserver.sqluri"] = sqluri - settings["tokenserver.node_url"] = public_url - settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}" - if "tokenserver.monkey_patch_gevent" not in settings: - # Default to no gevent monkey-patching - settings["tokenserver.monkey_patch_gevent"] = False - if "tokenserver.applications" not in settings: - # Default to just the sync-1.5 application - settings["tokenserver.applications"] = "sync-1.5" - if "tokenserver.secrets.backend" not in settings: - # Default to a single fixed signing secret - settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets" - settings["tokenserver.secrets.secrets"] = [secret] - if "tokenserver.allow_new_users" not in settings: - allow_new_users = settings.get("syncserver.allow_new_users") - if allow_new_users is not None: - settings["tokenserver.allow_new_users"] = allow_new_users - if "hawkauth.secrets.backend" not in settings: - # Default to the same secrets backend as the tokenserver - for key in settings.keys(): - if key.startswith("tokenserver.secrets."): - newkey = "hawkauth" + key[len("tokenserver"):] - settings[newkey] = settings[key] - if "storage.backend" not in settings: - # Default to sql syncstorage backend - settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage" - settings["storage.sqluri"] = sqluri - settings["storage.create_tables"] = True - if "browserid.backend" not in settings: - # Default to remote verifier, with base of public_url as only audience - audience = urlunparse(urlparse(public_url)._replace(path="")) - settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" - settings["browserid.audiences"] = audience - if "loggers" not in settings: - # Default to basic logging config. - root_logger = logging.getLogger("") - if not root_logger.handlers: - logging.basicConfig(level=logging.INFO) - - # Include the relevant sub-packages. - config.scan("syncserver") - config.include("syncstorage", route_prefix="/storage") - config.include("tokenserver", route_prefix="/token") - - # Add a top-level "it works!" view. - def itworks(request): - return Response("it works!") - - config.add_route('itworks', '/') - config.add_view(itworks, route_name='itworks') - - -@subscriber(NewRequest) -def reconcile_wsgi_environ_with_public_url(event): - """Event-listener that checks and tweaks WSGI environ based on public_url. - - This is a simple trick to help ensure that the configured public_url - matches the actual deployed address. It fixes fixes parts of the WSGI - environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts - that seem obviously mis-configured (e.g. http:// versus https://). - - It's very important to get public_url and WSGI environ matching exactly, - since they're used for browserid audience checking and HAWK signature - validation, so mismatches can easily cause strange and cryptic errors. - """ - request = event.request - public_url = request.registry.settings["syncserver.public_url"] - p_public_url = urlparse(public_url) - # If we don't have a SCRIPT_NAME, take it from the public_url. - # This is often the case if we're behind e.g. an nginx proxy that - # is serving us at some sub-path. - if not request.script_name: - request.script_name = p_public_url.path.rstrip("/") - # If the environ does not match public_url, requests are almost certainly - # going to fail due to auth errors. We can either bail out early, or we - # can forcibly clobber the WSGI environ with the values from public_url. - # This is a security risk if you've e.g. mis-configured the server, so - # it's not enabled by default. - application_url = request.application_url - if public_url != application_url: - if not request.registry.settings.get("syncserver.force_wsgi_environ"): - msg = "\n".join(( - "The public_url setting doesn't match the application url.", - "This will almost certainly cause authentication failures!", - " public_url setting is: %s" % (public_url,), - " application url is: %s" % (application_url,), - "You can disable this check by setting the force_wsgi_environ", - "option in your config file, but do so at your own risk.", - )) - logger.error(msg) - raise _JSONError([msg], status_code=500) - request.scheme = p_public_url.scheme - request.host = p_public_url.netloc - request.script_name = p_public_url.path.rstrip("/") - - -def get_configurator(global_config, **settings): - """Load a SyncStorge configurator object from deployment settings.""" - config = mozsvc.config.get_configurator(global_config, **settings) - config.begin() - try: - config.include(includeme) - finally: - config.end() - return config - - -def main(global_config, **settings): - """Load a SyncStorage WSGI app from deployment settings.""" - config = get_configurator(global_config, **settings) - return config.make_wsgi_app() diff --git a/sources/syncserver/staticnode.py b/sources/syncserver/staticnode.py deleted file mode 100644 index 560d3ac..0000000 --- a/sources/syncserver/staticnode.py +++ /dev/null @@ -1,221 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Simple node-assignment backend using a single, static node. - -This is a greatly-simplified node-assignment backend. It keeps user records -in an SQL database, but does not attempt to do any node management. All users -are implicitly assigned to a single, static node. - -XXX TODO: move this into the tokenserver repo. - -""" -import time -import urlparse -from mozsvc.exceptions import BackendError - -from sqlalchemy import Column, Integer, String, BigInteger, Index -from sqlalchemy import create_engine, Table, MetaData -from sqlalchemy.pool import QueuePool -from sqlalchemy.sql import text as sqltext -from sqlalchemy.exc import IntegrityError - -from tokenserver.assignment import INodeAssignment -from zope.interface import implements - - -metadata = MetaData() - - -users = Table( - "users", - metadata, - Column("uid", Integer(), primary_key=True, autoincrement=True, - nullable=False), - Column("service", String(32), nullable=False), - Column("email", String(255), nullable=False), - Column("generation", BigInteger(), nullable=False), - Column("client_state", String(32), nullable=False), - Column("created_at", BigInteger(), nullable=False), - Column("replaced_at", BigInteger(), nullable=True), - Index('lookup_idx', 'email', 'service', 'created_at'), - Index('clientstate_idx', 'email', 'service', 'client_state', unique=True), -) - - -_GET_USER_RECORDS = sqltext("""\ -select - uid, generation, client_state -from - users -where - email = :email -and - service = :service -order by - created_at desc, uid desc -limit - 20 -""") - - -_CREATE_USER_RECORD = sqltext("""\ -insert into - users - (service, email, generation, client_state, created_at, replaced_at) -values - (:service, :email, :generation, :client_state, :timestamp, NULL) -""") - - -_UPDATE_GENERATION_NUMBER = sqltext("""\ -update - users -set - generation = :generation -where - service = :service and email = :email and - generation < :generation and replaced_at is null -""") - - -_REPLACE_USER_RECORDS = sqltext("""\ -update - users -set - replaced_at = :timestamp -where - service = :service and email = :email - and replaced_at is null and created_at < :timestamp -""") - - -def get_timestamp(): - """Get current timestamp in milliseconds.""" - return int(time.time() * 1000) - - -class StaticNodeAssignment(object): - implements(INodeAssignment) - - def __init__(self, sqluri, node_url, **kw): - self.sqluri = sqluri - self.node_url = node_url - self.driver = urlparse.urlparse(sqluri).scheme.lower() - sqlkw = { - "logging_name": "syncserver", - "connect_args": {}, - "poolclass": QueuePool, - "pool_reset_on_return": True, - } - if self.driver == "sqlite": - # We must mark it as safe to share sqlite connections between - # threads. The pool will ensure there's on race conditions. - sqlkw["connect_args"]["check_same_thread"] = False - # If using a :memory: database, we must use a QueuePool of size - # 1 so that a single connection is shared by all threads. - if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"): - sqlkw["pool_size"] = 1 - sqlkw["max_overflow"] = 0 - if "mysql" in self.driver: - # Guard against the db closing idle conections. - sqlkw["pool_recycle"] = 3600 - self._engine = create_engine(sqluri, **sqlkw) - users.create(self._engine, checkfirst=True) - - def get_user(self, service, email): - params = {'service': service, 'email': email} - res = self._engine.execute(_GET_USER_RECORDS, **params) - try: - row = res.fetchone() - if row is None: - return None - # The first row is the most up-to-date user record. - user = { - 'email': email, - 'uid': row.uid, - 'node': self.node_url, - 'generation': row.generation, - 'client_state': row.client_state, - 'old_client_states': {} - } - # Any subsequent rows are due to old client-state values. - row = res.fetchone() - while row is not None: - user['old_client_states'][row.client_state] = True - row = res.fetchone() - return user - finally: - res.close() - - def allocate_user(self, service, email, generation=0, client_state=''): - params = { - 'service': service, 'email': email, 'generation': generation, - 'client_state': client_state, 'timestamp': get_timestamp() - } - try: - res = self._engine.execute(_CREATE_USER_RECORD, **params) - except IntegrityError: - raise - return self.get_user(service, email) - else: - res.close() - return { - 'email': email, - 'uid': res.lastrowid, - 'node': self.node_url, - 'generation': generation, - 'client_state': client_state, - 'old_client_states': {} - } - - def update_user(self, service, user, generation=None, client_state=None): - if client_state is None: - # uid can stay the same, just update the generation number. - if generation is not None: - params = { - 'service': service, - 'email': user['email'], - 'generation': generation, - } - res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params) - res.close() - user['generation'] = max(generation, user['generation']) - else: - # reject previously-seen client-state strings. - if client_state == user['client_state']: - raise BackendError('previously seen client-state string') - if client_state in user['old_client_states']: - raise BackendError('previously seen client-state string') - # need to create a new record for new client_state. - if generation is not None: - generation = max(user['generation'], generation) - else: - generation = user['generation'] - now = get_timestamp() - params = { - 'service': service, 'email': user['email'], - 'generation': generation, 'client_state': client_state, - 'timestamp': now, - } - try: - res = self._engine.execute(_CREATE_USER_RECORD, **params) - except IntegrityError: - user.update(self.get_user(service, user['email'])) - else: - self.get_user(service, user['email']) - user['uid'] = res.lastrowid - user['generation'] = generation - user['old_client_states'][user['client_state']] = True - user['client_state'] = client_state - res.close() - # mark old records as having been replaced. - # if we crash here, they are unmarked and we may fail to - # garbage collect them for a while, but the active state - # will be undamaged. - params = { - 'service': service, 'email': user['email'], 'timestamp': now - } - res = self._engine.execute(_REPLACE_USER_RECORDS, **params) - res.close() diff --git a/sources/syncserver/tests.ini b/sources/syncserver/tests.ini deleted file mode 100644 index fc9ea4b..0000000 --- a/sources/syncserver/tests.ini +++ /dev/null @@ -1,19 +0,0 @@ -[server:main] -use = egg:gunicorn -host = 0.0.0.0 -port = 5000 -workers = 1 -timeout = 30 - -[app:main] -use = egg:SyncServer - -[syncserver] -# This must be edited to point to the public URL of your server. -public_url = http://localhost:5000/ - -# This defines the database in which to store all server data. -#sqluri = sqlite:////tmp/syncserver.db - -# This is a secret key used for signing authentication tokens. -#secret = INSERT_SECRET_KEY_HERE