Index: ow/migrate.py
===================================================================
--- ow/migrate.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/migrate.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,155 @@
+import pkgutil
+import importlib
+import logging
+from bisect import insort, bisect_right
+
+from pyramid_zodbconn import get_connection
+from pyramid.scripting import prepare
+from pyramid.paster import bootstrap
+from pyramid.events import subscriber, ApplicationCreated
+
+from transaction import commit
+
+# Cache for maximum versions of migrations for packages
+MAX_CACHE = {}
+log = logging.getLogger(__name__)
+
+
+def get_indexes(package_name):
+    """
+    Gets a sorted list of migrations in a package
+    """
+    package = importlib.import_module(package_name)
+    indexes = []
+    for (loader, name, ispkg) in pkgutil.iter_modules(package.__path__):
+        try:
+            insort(indexes, int(name))
+        except ValueError:
+            continue
+    return indexes
+
+
+def get_max(package_name):
+    """
+    Get the maximum version for a package of migrations
+    """
+    if package_name in MAX_CACHE:
+        return MAX_CACHE[package_name]
+
+    indexes = get_indexes(package_name)
+    retval = 0
+    if len(indexes) > 0:
+        retval = indexes[-1]
+
+    MAX_CACHE[package_name] = retval
+    return retval
+
+
+def set_max_version(zodb_root, package_name):
+    """
+    Set the version to the maximum in the zodb_root using the given package
+    name
+    """
+    zodb_root = set_version(zodb_root, get_max(package_name))
+    return zodb_root
+
+
+def reset_version(request, version):
+    """
+    Forces the database version to a specific one and commits
+    """
+    dbroot = get_connection(request).root()
+    set_version(dbroot, version)
+    commit()
+
+
+def set_version(zodb_root, version):
+    """
+    Sets the version
+    """
+    zodb_root['database_version'] = version
+    return zodb_root
+
+
+def run_migrations(request, root, package_name):
+    """
+    Run migrations from a package_name
+    """
+    indexes = get_indexes(package_name)
+    dbroot = get_connection(request).root()
+    current = dbroot.get('database_version', 0)
+    migrations_to_apply = indexes[bisect_right(indexes, current):]
+    if len(migrations_to_apply) == 0:
+        log.info("Your database is in the latest version: '%i'. No migrations"
+                 " will be applied." % current)
+    else:
+        log.info("Starting migrations from %i" % current)
+    for index in migrations_to_apply:
+        migration = importlib.import_module("%s.%s" % (package_name, index))
+        if not hasattr(migration, 'migrate'):
+            log.error('No migrate method found for %s' % migration.__name__)
+            return False
+        log.info("Running migration %i" % index)
+        if hasattr(migration.migrate, '__doc__'):
+            doc = migration.migrate.__doc__.strip().split('\n')
+            description = []
+            for line in doc:
+                line = line.strip()
+                if line == '':
+                    break
+                description.append(line)
+            first_paragraph = ' '.join(description)
+            log.info('"""%s"""' % first_paragraph)
+        migration.migrate(root)
+        set_version(dbroot, index)
+        commit()
+
+
+def closer_wrapper(env):
+    closer = env['closer']
+    registry = env['registry']
+    root_factory = env['root_factory']
+    package = registry.settings.get(
+        'migrations_package', root_factory.__module__ + '.migrations')
+    # Run migrations
+    try:
+        run_migrations(env['request'], env['root'], package)
+    finally:
+        closer()
+
+
+def output(*args):  # pragma: nocover
+    print(args)
+
+
+def command_line_main(argv):
+    """
+    migrates the given database (Command line)
+
+    This command will use the bootstrap system from pyramid to start the
+    appllication without webserver.
+
+    Use this function to create a command line caller for the migrations. Pass
+    the sys.argv as parameter.
+    """
+    # Arguments
+    args = argv[1:]
+    if not len(args) == 1:
+        output('You must provide at least one argument: configuration file')
+        return 1
+    config_uri = args[0]
+    # Prepare application
+    env = bootstrap(config_uri)
+    closer_wrapper(env)
+    return 0
+
+
+@subscriber(ApplicationCreated)
+def application_created(event):
+    app = event.app
+    env = prepare(registry=app.registry)
+    closer_wrapper(env)
+
+
+def includeme(config):
+    config.scan('ow.migrate')
Index: ow/migrations/README.rst
===================================================================
--- ow/migrations/README.rst	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/migrations/README.rst	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,62 @@
+ZODB migrations
+===============
+
+This directory contains the migrations applied to the database when the app
+is starting.
+
+.. contents::
+
+
+Adding a new migration
+----------------------
+
+Migrations are named numerically, before adding a new migration look at the
+biggest number already available (look at **.py** files), then create a new
+**.py** file containing your migration.
+
+For example, if the last migration is **11.py**, create a file called **12.py**
+with your migration code inside.
+
+The migration file needs a **migrate** function in it, that accepts one
+parameter, the **root** of the pyramid application. There is where your code
+should modify whatever needs to be modified in the database.
+
+For example, this migration adds a new attribute to all objects in the
+database::
+
+  from BTrees.OOBTree import OOTreeSet
+
+  def migrate(root):
+      """
+      Adds some_new_attribute to all objects that do not have such attribute
+
+      >>> from models.root import Root, Child
+      >>> root = Root()
+      >>> child = Child()
+      >>> delattr(child, 'some_new_attr')
+      >>> assert getattr(child, 'some_new_attr', None) is None
+      True
+      >>> root.add(child)
+      >>> c = root.values()[0]
+      >>> assert getattr(c, 'some_new_attr', None) is None
+      True
+      >>> migrate(root)
+      >>> c = root.values()[0]
+      >>> assert getattr(c, 'some_new_attr', None) is None
+      False
+      """
+      for child in root.values():
+          if getattr(child, 'some_new_attr', None) is None:
+              child.some_new_attr = OOTreeSet()
+
+
+It is important that you add a proper doctest for the migration, because:
+
+1. The first paragraph of the migration will be shown on the app logs, letting
+   us known which kind of changes have been made to the database on start.
+
+2. Doctests there allow us to keep code coverage high and test those migrations
+   separately
+
+**Please use previous migrations for reference when in doubt on how to write
+a new migration**
Index: ow/migrations/__init__.py
===================================================================
--- ow/migrations/__init__.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/migrations/__init__.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,1 @@
+# ZODB migrations directory
Index: ow/tests/migrations/1.py
===================================================================
--- ow/tests/migrations/1.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/tests/migrations/1.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,10 @@
+from ow.migrate import output
+
+
+def migrate(root):
+    """
+    Fake first migration
+
+    Super fake migration
+    """
+    output('Fake first migration')
Index: ow/tests/migrations/2.py
===================================================================
--- ow/tests/migrations/2.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/tests/migrations/2.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,8 @@
+from ow.migrate import output
+
+
+def migrate(root):
+    """
+    Fake second migration
+    """
+    output('Fake second migration')
Index: ow/tests/test_migrate.py
===================================================================
--- ow/tests/test_migrate.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
+++ ow/tests/test_migrate.py	(revision eb20dc818a798cb30bba9c9f57b4e372ab06e03c)
@@ -0,0 +1,168 @@
+import os
+from shutil import copyfile
+from unittest.mock import patch, Mock
+
+import pytest
+
+from ow import migrate
+
+
+class MigrateUnitTests(object):
+    def test_includeme(self):
+        config = Mock()
+        migrate.includeme(config)
+        config.scan.assert_called_with('ow.migrate')
+
+    @patch('ow.migrate.run_migrations')
+    def test_closer_wrapper_ok(self, run):
+        closer = Mock()
+        env = dict(
+            registry=Mock(settings={}),
+            root_factory=Mock(__module__='mytest'),
+            request=1,
+            root=2,
+            closer=closer)
+        migrate.closer_wrapper(env)
+        run.assert_called_with(1, 2, 'mytest.migrations')
+        assert closer.called
+
+    @patch('ow.migrate.closer_wrapper')
+    @patch('ow.migrate.prepare')
+    def test_application_created_ok(self, prepare, wrap):
+        event = Mock()
+        migrate.application_created(event)
+        assert prepare.called
+        assert wrap.called
+
+    @patch('ow.migrate.output')
+    def test_command_line_no_conf(self, pr):
+        ret = migrate.command_line_main(['test.py'])
+        assert ret == 1
+        assert pr.called
+
+    @patch('ow.migrate.closer_wrapper')
+    @patch('ow.migrate.bootstrap')
+    def test_command_line_no(self, bs, wrap):
+        ret = migrate.command_line_main(['test.py', 'dev.ini'])
+        assert ret == 0
+        assert bs.called
+        assert wrap.called
+
+    @patch('ow.migrate.commit')
+    @patch('ow.migrate.get_connection')
+    def test_reset_version(self, pget_connection, pcommit):
+        zodb = {}
+        pget_connection.return_value.root.return_value = zodb
+        migrate.reset_version('myrequest', 25)
+        assert zodb == {'database_version': 25}
+        pcommit.asert_called_with()
+
+
+def cleanup():
+    """
+    Clean up pyc files generated while running the following test suite
+    """
+    migrations = os.path.join(os.path.dirname(__file__), 'migrations')
+    for f in os.listdir(migrations):
+        if '.pyc' in f or 'fail' in f or f == '3.py':
+            os.remove(os.path.join(migrations, f))
+
+
+class mocked_get_connection(object):
+    """
+    This is a class we can use to mock pyramid_zodbconn.get_connection()
+    (see test_run_migrations)
+    """
+    def __init__(self, versions=0):
+        self.versions = versions
+
+    def root(self):
+        return {'database_version': self.versions}
+
+
+class MigrateTests(object):
+
+    package_name = 'ow.tests.migrations'
+    ini_path = os.path.join(os.path.dirname(__file__), 'migrations')
+
+    def test_get_indexes_ok(self):
+        indexes = migrate.get_indexes(self.package_name)
+        assert isinstance(indexes, list)
+        assert len(indexes) == 2
+
+    def test_get_indexes_fail(self):
+        migrate.get_indexes(self.package_name)
+        with pytest.raises(ImportError):
+            migrate.get_indexes('nonexistent.module.migrations')
+
+    def test_get_indexes_invalid(self):
+        # Create a new migration file with an invalid name, so the get_indexes
+        # will raise a ValueError exception
+        copyfile(os.path.join(os.path.dirname(__file__), 'migrations/1.py'),
+                 os.path.join(os.path.dirname(__file__), 'migrations/fail.py'))
+        indexes = migrate.get_indexes(self.package_name)
+        assert isinstance(indexes, list)
+        assert len(indexes) == 2
+
+    def test_get_max_in_max_cache(self):
+        with patch.dict(migrate.MAX_CACHE, {self.package_name: 10}):
+            max_version = migrate.get_max(self.package_name)
+            assert max_version == 10
+
+    def test_get_max(self):
+        max_version = migrate.get_max(self.package_name)
+        assert max_version == 2
+
+    def test_version(self):
+        # instead of a real ZODB root, we do use a simple dict here,
+        # it should be enough for what need to test.
+        root = {}
+        root = migrate.set_version(root, 10)
+        assert root['database_version'] == 10
+
+    def test_max_version(self):
+        # instead of a real ZODB root, we do use a simple dict here,
+        # it should be enough for what need to test.
+        root = {}
+        root = migrate.set_max_version(root, self.package_name)
+        assert root['database_version'] == 2
+
+    @patch('ow.migrate.get_connection')
+    @patch('ow.tests.migrations.1.output')
+    @patch('ow.tests.migrations.2.output')
+    def test_run_all_migrations(self, pr2, pr1, gc):
+        """
+        Test that all migrations apply
+        """
+        gc.return_value = mocked_get_connection()
+        migrate.run_migrations(None, {}, self.package_name)
+        cleanup()
+        assert pr1.called
+        assert pr2.called
+
+    @patch('ow.migrate.get_connection')
+    def test_run_no_migrations(self, gc):
+        """
+        Test that there are no more migrations to apply
+        """
+        gc.return_value = mocked_get_connection(versions=2)
+        migrate.run_migrations(None, {}, self.package_name)
+
+    @patch('ow.migrate.get_connection')
+    @patch('ow.tests.migrations.1.output')
+    @patch('ow.tests.migrations.2.output')
+    def test_run_invalid_migrations(self, pr2, pr1, gc):
+        """
+        Test what happens if a migration does not contains the proper migrate
+        method
+        """
+        invalid_migration = open(os.path.join(os.path.dirname(__file__),
+                                              'migrations/3.py'), 'w')
+        invalid_migration.write('# This is an empty migration, just for tests')
+        invalid_migration.write('def no_migrate_method_here():')
+        invalid_migration.write('    print "Nothing to see here!"')
+        invalid_migration.close()
+        gc.return_value = mocked_get_connection(versions=0)
+        migrate.run_migrations(None, {}, self.package_name)
+        assert pr1.called
+        assert pr2.called
