Add a minimal test suite for the data store operating on public DBus API level. Checks all public API calls, including some simple performance measurements.
Signed-off-by: Sascha Silbe <sascha-...@silbe.org> create mode 100644 tests/.gitignore create mode 100644 tests/Makefile create mode 100644 tests/__init__.py create mode 100644 tests/basic_api_v2.txt create mode 100755 tests/runalltests.py create mode 100644 tests/test_massops.py create mode 100644 tests/test_migration_v1_v2.py Signed-off-by: Sascha Silbe <si...@activitycentral.com> --- Makefile.am | 6 +- tests/.gitignore | 1 + tests/Makefile | 20 +++ tests/__init__.py | 1 + tests/basic_api_v2.txt | 135 +++++++++++++++ tests/runalltests.py | 360 +++++++++++++++++++++++++++++++++++++++++ tests/test_massops.py | 175 ++++++++++++++++++++ tests/test_migration_v1_v2.py | 175 ++++++++++++++++++++ 8 files changed, 868 insertions(+), 5 deletions(-) create mode 100644 tests/.gitignore create mode 100644 tests/Makefile create mode 100644 tests/__init__.py create mode 100644 tests/basic_api_v2.txt create mode 100755 tests/runalltests.py create mode 100644 tests/test_massops.py create mode 100644 tests/test_migration_v1_v2.py diff --git a/Makefile.am b/Makefile.am index bfebefe..d450f24 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,3 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = bin etc src - -test: - @cd tests - $(MAKE) -C tests test +SUBDIRS = bin etc src tests diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..2460008 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +!Makefile diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..cf9ac60 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,20 @@ +all: +install: +uninstall: + +check: + @./runalltests.py + +valgrind: + @echo "Profiling the process. Run kcachegrind on the output" + valgrind --tool=callgrind --suppressions=valgrind-python.supp python runalltests.py + +distclean: clean +clean: + @find . -name "*.pyc" -exec rm {} \; + @find . -name "*.pyo" -exec rm {} \; + @find . -name "*~" -exec rm {} \; + @find . -name "callgrind.out*" -exec rm {} \; + +tags: + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..5b3912c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# testing package diff --git a/tests/basic_api_v2.txt b/tests/basic_api_v2.txt new file mode 100644 index 0000000..15d4cd6 --- /dev/null +++ b/tests/basic_api_v2.txt @@ -0,0 +1,135 @@ +>>> import os +>>> import tempfile +>>> import time + +Define some helper functions +>>> def test_unique(items): +... return not [True for e in items if items.count(e) > 1] +>>> def to_native(value): +... if isinstance(value, list): +... return [to_native(e) for e in value] +... elif isinstance(value, dict): +... return dict([(to_native(k), to_native(v)) for k, v in value.items()]) +... elif isinstance(value, unicode): +... return unicode(value) +... elif isinstance(value, str): +... return str(value) +... return value + + +Connect to datastore using DBus and wait for it to get ready: +>>> import dbus +>>> DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +>>> DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +>>> DS_DBUS_PATH = '/org/laptop/sugar/DataStore' +>>> bus = dbus.SessionBus() +>>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) + + +Make sure we're starting from an empty datastore: +>>> assert ds.find({}, [], byte_arrays=True) == ([], 0) + + +Create something to play with: +>>> o1_uid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, '', False) +>>> assert isinstance(o1_uid, basestring) +>>> o2_uid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False) +>>> assert isinstance(o2_uid, basestring) +>>> o3_uid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False) +>>> assert isinstance(o3_uid, basestring) +>>> assert test_unique([o1_uid, o2_uid, o3_uid]) + + +Check everything is there: +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 1', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest2'}, +... {u'title': 'DS test object 3', u'activity': 'org.sugarlabs.DataStoreTest2'}] +>>> ds.get_filename(o1_uid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_filename(o2_uid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_filename(o3_uid, byte_arrays=True) +dbus.String(u'') + + + +Test get_uniquevaluesfor(). +>>> sorted(ds.get_uniquevaluesfor('activity', {})) +[dbus.String(u'org.sugarlabs.DataStoreTest1'), dbus.String(u'org.sugarlabs.DataStoreTest2')] + + +Change some entries: +>>> ds.update(o1_uid, {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'foo'}, '', False) +>>> ds.update(o2_uid, {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'bar baz'}, '', False) +>>> ds.update(o3_uid, {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.sugarlabs.DataStoreTest3', 'timestamp': 10000}, '', False) +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] + +Retrieve metadata for a single entry, ignoring variable data: +>>> d=dict(ds.get_properties(o3_uid, byte_arrays=True)) +>>> del d['uid'], d['timestamp'], d['creation_time'] +>>> assert to_native(d) == {u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'} + + +Find entries using "known" metadata: +>>> assert sorted(to_native(ds.find({'mime_type': ['text/plain']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}, +... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}] +>>> assert sorted(to_native(ds.find({'mime_type': ['text/html']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'uid': o3_uid}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'timestamp': (9000, 11000)}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] + +Find entries using "unknown" metadata (=> returns all entries): +>>> assert sorted(to_native(ds.find({'title': 'DS test object 2'}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}, +... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}] + +You can specify a (primary) sort order. Please note that the secondary sort order is undefined / implementation-dependent. +>>> assert to_native(ds.find({'order_by': ['+title']}, ['title', 'activity'], byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}] +>>> assert to_native(ds.find({'order_by': ['-title']}, ['title', 'activity'], byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] + +Delete an entry: +>>> ds.delete(o1_uid) +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest3'}] + + +Create an entry with content: +>>> dog_content = 'The quick brown dog jumped over the lazy fox.' +>>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'} +>>> dog_file = tempfile.NamedTemporaryFile() +>>> dog_file.write(dog_content) +>>> dog_file.flush() +>>> dog_uid = ds.create(dog_props, dog_file.name, False) + +Retrieve and verify the entry with content: +>>> dog_retrieved = ds.get_filename(dog_uid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) + +Update the entry content: +>>> dog_content = 'The quick brown fox jumped over the lazy dog.' +>>> dog_file.seek(0) +>>> dog_file.write(dog_content) +>>> dog_file.flush() +>>> ds.update(dog_uid, dog_props, dog_file.name, False) + +Verify updated content: +>>> dog_retrieved = ds.get_filename(dog_uid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) +>>> dog_file.close() diff --git a/tests/runalltests.py b/tests/runalltests.py new file mode 100755 index 0000000..9a7c8f8 --- /dev/null +++ b/tests/runalltests.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python +"""Run all tests in the current directory. + +You can either call it without arguments to run all tests or name specific +ones to run: + + ./runalltests.py test_massops.py +""" + +import doctest +import errno +import logging +from optparse import OptionParser +import os +import os.path +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +import dbus +import dbus.mainloop.glib +import gobject + + +logging.basicConfig(level=logging.WARN, + format='%(asctime)-15s %(name)s %(levelname)s:' + ' %(message)s', + stream=sys.stderr) + + +DOCTESTS = [ + 'basic_api_v2.txt', +] +DOCTEST_OPTIONS = doctest.ELLIPSIS +DOCTEST_OPTIONS |= doctest.REPORT_ONLY_FIRST_FAILURE + +DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +DS_DBUS_PATH = '/org/laptop/sugar/DataStore' + +ENVIRONMENT_WHITELIST = [ + 'LD_LIBRARY_PATH', + 'MALLOC_CHECK_', + 'MASSOPS_RUNS', + 'SUGAR_LOGGER_LEVEL', +] + +SERVICE_TEMPLATE = """ +[D-BUS Service] +Name = org.laptop.sugar.DataStore +Exec = %s/bin/datastore-service +""" + + +def setup(): + """Prepare for testing and return environment. + + Sets HOME and creates a new process group so we can clean up easily later. + Sets up environment variables and imports whitelisted ones. + """ + environment = {} + for name in ENVIRONMENT_WHITELIST: + if name in os.environ: + environment[name] = os.environ[name] + + environment['HOME'] = tempfile.mkdtemp(prefix='datastore-test') + if 'PYTHONPATH' in os.environ: + python_path = os.environ.get('PYTHONPATH').split(':') + else: + python_path = [] + + # Run tests on sources instead of on installed files. + basedir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), '..') + python_path = [os.path.join(basedir, 'src')] + python_path + environment['PYTHONPATH'] = ':'.join(python_path) + environment['PATH'] = ':'.join([os.path.join(basedir, 'bin'), + os.environ['PATH']]) + + service_dir = os.path.join(environment['HOME'], 'dbus-1', 'services') + service_path = os.path.join(service_dir, + 'org.laptop.sugar.DataStore.service') + os.makedirs(service_dir) + service_file = file(service_path, 'w') + service_file.write(SERVICE_TEMPLATE % (basedir, )) + service_file.close() + environment['XDG_DATA_DIRS'] = environment['HOME'] + + os.setpgid(0, 0) + # prevent suicide in cleanup() + signal.signal(signal.SIGTERM, signal.SIG_IGN) + return environment + + +def wait_children(): + """Wait for all children to exit.""" + try: + while True: + os.wait() + + except OSError, exception: + if exception.errno in (errno.ECHILD, errno.ESRCH): + # ECHILD is not documented in kill(2) and kill(2p), but used by + # Linux to indicate no child processes remaining to be waited for + return + + raise + + +def cleanup(home, keep_files, dbus_pid): + """Clean up test environment. + + Kills all children and removes home directory. + """ + if dbus_pid: + os.kill(-dbus_pid, signal.SIGTERM) + + os.kill(0, signal.SIGTERM) + wait_children() + + if not keep_files: + shutil.rmtree(home) + + +class TestSuiteWrapper(unittest.TestCase): + """Wrap a test suite to clean up after it. + + This ensures each test module gets a clean data store instance. + """ + + def __init__(self, suite): + self._wrapped_suite = suite + self._bus = dbus.SessionBus() + self._loop = None + unittest.TestCase.__init__(self) + + def runTest(self, result=None): + self._wrapped_suite(result) + + def run(self, result=None): + if result is None: + result = self.defaultTestResult() + result.startTest(self) + try: + try: + self.setUp() + except KeyboardInterrupt: + raise + except: + result.addError(self, self._exc_info()) + return + + ok = False + try: + self.runTest(result) + ok = True + except self.failureException: + result.addFailure(self, self._exc_info()) + except KeyboardInterrupt: + raise + except: + result.addError(self, self._exc_info()) + + try: + self.tearDown() + except KeyboardInterrupt: + raise + except: + result.addError(self, self._exc_info()) + ok = False + if ok: + result.addSuccess(self) + finally: + result.stopTest(self) + + def shortDescription(self): + doc = self._wrapped_suite.__doc__ + return doc and doc.split('\n')[0].strip() or None + + def tearDown(self): + self._kill_data_store() + self._clean_data_store() + + def _kill_data_store(self): + pgrep = subprocess.Popen(['pgrep', '-g', os.environ['DBUS_PID'], + '-f', 'datastore-service'], + close_fds=True, stdout=subprocess.PIPE) + stdout, stderr_ = pgrep.communicate() + pids = stdout.strip().split('\n') + if len(pids) != 1 or not pids[0]: + raise ValueError("Can't find (a single) data store process " + "(pgrep output %r)" % (stdout, )) + + pid = int(pids[0]) + self._loop = gobject.MainLoop() + self._bus.watch_name_owner(DS_DBUS_SERVICE, self._service_changed_cb) + os.kill(pid, signal.SIGTERM) + self._loop.run() + + def _service_changed_cb(self, new_owner): + if not new_owner: + self._loop.quit() + + def _clean_data_store(self): + profile = os.environ.get('SUGAR_PROFILE', 'default') + base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile) + root_path = os.path.join(base_dir, 'datastore') + shutil.rmtree(root_path) + + +class TimedTestResult(unittest._TextTestResult): + """Store and display test results and test runtime. + + Only displays actual tests, not test suite wrappers.""" + + # Depending on a private class is bad style, but the only alternative is + # copying it verbatim. + # pylint: disable=W0212 + + def __init__(self, stream, descriptions, verbosity): + unittest._TextTestResult.__init__(self, stream, descriptions, + verbosity) + self.start_times = {} + self.run_times = {} + + def startTest(self, test): + self.start_times[test] = time.time() + unittest.TestResult.startTest(self, test) + if not self.showAll: + return + + description = self.getDescription(test) + if isinstance(test, TestSuiteWrapper): + self.stream.write('Test Suite: %s\n' % (description, )) + else: + self.stream.write(' %s ... ' % (description, )) + + def stopTest(self, test): + if test in self.start_times and test not in self.run_times: + self.run_times[test] = time.time() - self.start_times[test] + + unittest._TextTestResult.stopTest(self, test) + + def addSuccess(self, test): + if test in self.start_times and test not in self.run_times: + self.run_times[test] = time.time() - self.start_times[test] + + run_time = self.run_times.get(test, -1) + + unittest.TestResult.addSuccess(self, test) + + if isinstance(test, TestSuiteWrapper): + return + + if self.showAll: + self.stream.writeln('ok (%.3fs)' % (run_time, )) + elif self.dots: + self.stream.write('.') + + +class TimedTestRunner(unittest.TextTestRunner): + """Run tests, displaying test result and runtime in textual form.""" + + def _makeResult(self): + return TimedTestResult(self.stream, self.descriptions, self.verbosity) + + +def test_suite(tests=None): + suite = unittest.TestSuite() + if not tests: + test_dir = os.path.dirname(__file__) + tests = DOCTESTS + tests += [name for name in os.listdir(test_dir) + if name.startswith('test') and name.endswith('.py')] + + for test in tests: + if test.endswith('.txt'): + doc_suite = doctest.DocFileSuite(test, optionflags=DOCTEST_OPTIONS) + doc_suite.__doc__ = test + suite.addTest(TestSuiteWrapper(doc_suite)) + + elif test.endswith('.py'): + m = __import__(test[:-3]) + if hasattr(m, 'suite'): + suite.addTest(TestSuiteWrapper(m.suite())) + + return suite + + +def run_tests(tests): + dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) + runner = TimedTestRunner(verbosity=2) + suite = test_suite(tests) + result = runner.run(suite) + if result.wasSuccessful(): + return 0 + else: + return 10 + + +def _start_dbus(environment): + pipe = subprocess.Popen(['dbus-launch'], stdout=subprocess.PIPE, + close_fds=True, env=environment, + cwd=environment['HOME']) + stdout, stderr_ = pipe.communicate() + pid = None + address = None + for line in stdout.strip().split('\n'): + key, value = line.split('=', 1) + if key == 'DBUS_SESSION_BUS_ADDRESS': + address = value + elif key == 'DBUS_SESSION_BUS_PID': + pid = int(value) + else: + raise ValueError('Cannot parse dbus-launch output: %r' % (line, )) + + assert pid is not None + assert address is not None + return pid, address + + +def _parse_options(): + """Parse command line arguments.""" + parser = OptionParser() + parser.add_option('-k', '--keep', dest='keep', + action='store_true', default=False, + help='Keep temporary files') + parser.add_option('', '--stage2', dest='stage2', + action='store_true', default=False, + help='For internal use only') + return parser.parse_args() + + +def main(my_name, arguments): + options, tests = _parse_options() + if not options.stage2: + environment = setup() + dbus_pid = None + dbus_address = None + try: + dbus_pid, dbus_address = _start_dbus(environment) + environment['DBUS_SESSION_BUS_ADDRESS'] = dbus_address + environment['DBUS_PID'] = str(dbus_pid) + + pipe = subprocess.Popen([os.path.abspath(my_name), + '--stage2'] + arguments, + cwd=environment['HOME'], env=environment) + return pipe.wait() + + finally: + cleanup(environment['HOME'], options.keep, dbus_pid) + + return run_tests(tests) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[0], sys.argv[1:])) diff --git a/tests/test_massops.py b/tests/test_massops.py new file mode 100644 index 0000000..edce0c3 --- /dev/null +++ b/tests/test_massops.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +"""Large number of operations intended for measuring performance.""" + +import dbus +import decorator +import os +import tempfile +import time +import unittest + + +DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +DS_DBUS_PATH = '/org/laptop/sugar/DataStore' +NUM_RUNS = int(os.environ.get('MASSOPS_RUNS', '100')) +IGNORE_PROPERTIES = [ + 'checksum', + 'creation_time', + 'number', + 'timestamp', + 'uid', +] + + +@decorator.decorator +def repeat(func, *args, **kwargs): + """Run the decorated function NUM_RUNS times.""" + for i_ in range(NUM_RUNS): + func(*args, **kwargs) + + +class MassOpsTestCase(unittest.TestCase): + """Large number of operations intended for measuring performance.""" + + def setUp(self): + # pylint: disable=C0103 + self._bus = dbus.SessionBus() + self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE, + DS_DBUS_PATH), DS_DBUS_INTERFACE) + + _create_properties = { + 'title': 'DS test object', + 'mime_type': 'text/plain', + 'activity': 'org.sugarlabs.DataStoreTest1', + } + _create_content = 'Foo bar\n' * 1000 + + def test_create(self): + """Run create() lots of times to create new objects.""" + for i in range(NUM_RUNS): + content_file = tempfile.NamedTemporaryFile() + content_file.write(self._create_content) + content_file.flush() + properties = self._create_properties.copy() + properties['number'] = str(i) + properties['timestamp'] = time.time() + self._datastore.create(properties, content_file.name, False) + content_file.close() + + @repeat + def test_find_all(self): + """Run find() to list all entries.""" + entries, total_count = self._datastore.find({}, ['number'], + byte_arrays=True) + self.assertEquals(total_count, NUM_RUNS) + self.assertEquals(total_count, len(entries)) + for position, entry in enumerate(entries): + self.assertEquals(int(entry['number']), NUM_RUNS - position - 1) + + @repeat + def test_find_all_reverse_time(self): + """Run find() to list all entries in reverse chronological order.""" + entries, total_count = \ + self._datastore.find({'order_by': ['-timestamp']}, ['number'], + byte_arrays=True) + self.assertEquals(total_count, NUM_RUNS) + self.assertEquals(total_count, len(entries)) + for position, entry in enumerate(entries): + self.assertEquals(int(entry['number']), position) + + @repeat + def test_find_all_title(self): + """Run find() to list all entries ordered by title.""" + entries, total_count = \ + self._datastore.find({'order_by': ['+title']}, ['tree_id'], + byte_arrays=True) + self.assertEquals(total_count, NUM_RUNS) + self.assertEquals(total_count, len(entries)) + + @repeat + def test_find_all_reverse_title(self): + """Run find() to list all entries ordered by title (reversed).""" + entries, total_count = \ + self._datastore.find({'order_by': ['-title']}, ['tree_id'], + byte_arrays=True) + self.assertEquals(total_count, NUM_RUNS) + self.assertEquals(total_count, len(entries)) + + @repeat + def test_find_all_chunked(self): + """Run find() to list all entries in small chunks.""" + chunk_size = 30 + for chunk_start in range(0, NUM_RUNS, 30): + entries, total_count = \ + self._datastore.find({'offset': chunk_start, + 'limit': chunk_size}, ['number'], + byte_arrays=True) + self.assertEquals(len(entries), + min(chunk_size, NUM_RUNS - chunk_start)) + self.assertEquals(total_count, NUM_RUNS) + for position, entry in enumerate(entries): + self.assertEquals(int(entry['number']), + NUM_RUNS - (chunk_start + position) - 1) + + def test_get_properties(self): + """Run get_properties() on all entries and verify result.""" + for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: + properties = \ + self._datastore.get_properties(entry['uid'], byte_arrays=True) + self.assertEquals(properties.pop('filesize'), + str(len(self._create_content))) + self._filter_properties(properties) + self.assertEquals(properties, self._create_properties) + + def test_get_filename(self): + """Run get_filename() on all entries and verify content.""" + for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: + filename = self._datastore.get_filename(entry['uid'], + byte_arrays=True) + try: + self.assertEquals(file(filename).read(), self._create_content) + finally: + os.remove(filename) + + _update_properties = { + 'title': 'DS test object (updated)', + 'mime_type': 'text/plain', + 'activity': 'org.sugarlabs.DataStoreTest1', + } + _update_content = 'Foo bar baz\n' * 1000 + + def test_update(self): + """Update the content of all existing entries""" + content_file = tempfile.NamedTemporaryFile() + content_file.write(self._update_content) + content_file.flush() + for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: + self._datastore.update(entry['uid'], self._update_properties, + content_file.name, False) + + def test_update_verify(self): + """ + Verify test_update() has changed content and metadata of all entries. + """ + for entry in self._datastore.find({}, [], byte_arrays=True)[0]: + filename = self._datastore.get_filename(entry['uid'], + byte_arrays=True) + self.assertEquals(entry.pop('filesize'), + str(len(self._update_content))) + self._filter_properties(entry) + try: + self.assertEquals(entry, self._update_properties) + self.assertEquals(file(filename).read(), self._update_content) + finally: + os.remove(filename) + + def _filter_properties(self, properties): + for key in IGNORE_PROPERTIES: + properties.pop(key, None) + + +def suite(): + test_suite = unittest.TestLoader().loadTestsFromTestCase(MassOpsTestCase) + test_suite.__doc__ = MassOpsTestCase.__doc__ + return test_suite diff --git a/tests/test_migration_v1_v2.py b/tests/test_migration_v1_v2.py new file mode 100644 index 0000000..b574fd8 --- /dev/null +++ b/tests/test_migration_v1_v2.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +"""Test datastore migration from version 1 to version 2.""" + +import dbus +import hashlib +import os +import time +import unittest +import uuid + + +DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +DS_DBUS_PATH = '/org/laptop/sugar/DataStore' +IGNORE_PROPERTIES = [ + 'activity_id', + 'checksum', + 'creation_time', + 'ctime', + 'mtime', + 'number', + 'timestamp', + 'uid', +] + + +class MigrationV1V2TestCase(unittest.TestCase): + """Test datastore migration from version 1 to version 2.""" + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self._templates = self._v1_properties * 10 + + def setUp(self): + # pylint: disable=C0103 + profile = os.environ.get('SUGAR_PROFILE', 'default') + base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile) + self._root_path = os.path.join(base_dir, 'datastore') + if not os.path.exists(self._root_path): + self._create_v1_datastore() + + self._bus = dbus.SessionBus() + self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE, + DS_DBUS_PATH), + DS_DBUS_INTERFACE) + + _v1_properties = [ + { + 'title': lambda number: 'DS test object %d' % (number, ), + 'mime_type': 'text/plain', + }, + { + 'title': lambda number: 'DS test object %d' % (number, ), + 'mime_type': 'text/html', + }, + { + 'title': lambda number: 'DS test object %d' % (number, ), + 'title_set_by_user': '1', + 'keep': '1', + 'mime_type': 'text/html', + 'activity': 'org.sugarlabs.DataStoreTest2', + 'activity_id': lambda number_: str(uuid.uuid4()), + 'timestamp': lambda number_: time.time(), + 'icon-color': '#00ff00,#0000ff', + 'buddies': '{}', + 'description': 'DS migration test object', + 'tags': lambda number: 'test tag%d' % (number, ), + 'preview': dbus.ByteArray(''.join([chr(i) for i in range(255)])), + }, + { + 'title': lambda number: 'DS test object %d' % (number, ), + 'activity': 'org.sugarlabs.DataStoreTest3', + 'activity_id': lambda number_: str(uuid.uuid4()), + 'ctime': lambda number_: time.strftime('%Y-%m-%dT%H:%M:%S'), + }, + { + 'title': lambda number: 'DS test object %d' % (number, ), + 'activity': 'org.sugarlabs.DataStoreTest4', + 'activity_id': lambda number_: str(uuid.uuid4()), + 'mtime': lambda number_: time.strftime('%Y-%m-%dT%H:%M:%S'), + }, + {}, + ] + + def _v1_content(self, num): + return ('Foo bar %d\n' % (num, )) * 1000 + + def _create_v1_datastore(self): + """Create a version 1 datastore on disk.""" + os.makedirs(self._root_path) + file(os.path.join(self._root_path, 'version'), 'w').write('1') + for i, template in enumerate(self._templates): + metadata = self._fill_template(template, i) + data = self._v1_content(i) + tree_id = str(uuid.uuid4()) + metadata['uid'] = tree_id + metadata['number'] = i + + self._create_v1_entry(tree_id, metadata, data) + + def _fill_template(self, template, i): + metadata = {} + for (key, value) in template.items(): + if callable(value): + value = value(i) + + metadata[key] = value + + return metadata + + def _create_v1_entry(self, tree_id, metadata, data): + """Create a single version 1 datastore entry.""" + checksum = hashlib.md5(data).hexdigest() + entry_dir = os.path.join(self._root_path, tree_id[:2], tree_id) + os.makedirs(entry_dir) + file(os.path.join(entry_dir, 'data'), 'w').write(data) + self._write_v1_metadata(os.path.join(entry_dir, 'metadata'), metadata) + checksum_dir = os.path.join(self._root_path, 'checksums', checksum) + os.makedirs(checksum_dir) + file(os.path.join(checksum_dir, tree_id), 'w').close() + + def _write_v1_metadata(self, directory, metadata): + os.makedirs(directory) + for key, value in metadata.items(): + file(os.path.join(directory, key), 'w').write(str(value)) + + def test_000_wait_ready(self): + """Wait for data store to finish initialisation (including migration). + """ + self._datastore.wait_ready() + + def test_find_all(self): + """Run find() to list all migrated entries.""" + entries_, count = self._find({}, ['uid']) + self.assertEquals(count, len(self._templates)) + + def test_get_properties(self): + """Run get_properties() on all entries and verify result.""" + for entry in self._find({}, ['uid'])[0]: + properties = self._datastore.get_properties(entry['uid'], + byte_arrays=True) + number = int(properties['number']) + expected = self._fill_template(self._templates[number], + number) + self.assertEquals(properties.pop('filesize'), + str(len(self._v1_content(number)))) + self._filter_properties(properties) + self._filter_properties(expected) + self.assertEquals(properties, expected) + + def test_get_filename(self): + """Run get_filename() on all entries and verify content.""" + for entry in self._find({}, ['number', 'uid'])[0]: + filename = self._datastore.get_filename(entry['uid'], + byte_arrays=True) + content = file(filename).read() + os.remove(filename) + number = int(entry['number']) + expected = self._v1_content(number) + self.assertEquals(content, expected) + + def _find(self, query, properties): + return self._datastore.find(dbus.Dictionary(query, signature='sv'), + properties, byte_arrays=True) + + def _filter_properties(self, properties): + for key in IGNORE_PROPERTIES: + properties.pop(key, None) + + +def suite(): + test_loader = unittest.TestLoader() + test_suite = test_loader.loadTestsFromTestCase(MigrationV1V2TestCase) + test_suite.__doc__ = MigrationV1V2TestCase.__doc__ + return test_suite -- 1.7.2.3 _______________________________________________ Sugar-devel mailing list Sugar-devel@lists.sugarlabs.org http://lists.sugarlabs.org/listinfo/sugar-devel