tloubrieu-jpl commented on a change in pull request #1:
URL:
https://github.com/apache/incubator-sdap-ingester/pull/1#discussion_r441176767
##########
File path: collection_manager/tests/services/test_CollectionWatcher.py
##########
@@ -0,0 +1,203 @@
+import os
+import tempfile
+import time
+import unittest
+from datetime import datetime
+from unittest.mock import Mock
+
+from collection_manager.entities import Collection
+from collection_manager.entities.exceptions import YamlParsingError,
CollectionConfigFileNotFoundError, \
+ RelativePathCollectionError, ConflictingPathCollectionError
+from collection_manager.services import CollectionWatcher
+
+
+class TestCollectionWatcher(unittest.TestCase):
+
+ def test_collections_returns_all_collections(self):
+ collection_watcher = CollectionWatcher('/foo', Mock(), Mock())
+ collection_watcher._collections_by_dir = {
+ "/foo": {
+ Collection("id1", "var1", "path1", 1, 2, datetime.now(),
datetime.now()),
+ Collection("id2", "var2", "path2", 3, 4, datetime.now(),
datetime.now()),
+ },
+ "/bar": {
+ Collection("id3", "var3", "path3", 5, 6, datetime.now(),
datetime.now()),
+ Collection("id4", "var4", "path4", 7, 8, datetime.now(),
datetime.now()),
+ }
+ }
+ flattened_collections = collection_watcher.collections()
+ self.assertEqual(len(flattened_collections), 4)
+
+ def test_load_collections_loads_all_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+ collection_watcher._load_collections()
+
+ self.assertEquals(len(collection_watcher._collections_by_dir), 2)
+
self.assertEquals(len(collection_watcher._collections_by_dir['/opt/data/grace']),
2)
+
self.assertEquals(len(collection_watcher._collections_by_dir['/opt/data/avhrr']),
1)
+
+ def test_load_collections_with_bad_yaml_syntax(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections_bad.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ self.assertRaises(YamlParsingError,
collection_watcher._load_collections)
+
+ def test_load_collections_with_file_not_found(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/does_not_exist.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ self.assertRaises(CollectionConfigFileNotFoundError,
collection_watcher._load_collections)
+
+ def test_get_updated_collections_returns_all_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ updated_collections = collection_watcher._get_updated_collections()
+ self.assertSetEqual(updated_collections,
collection_watcher.collections())
+
+ def test_get_updated_collections_returns_no_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+ collection_watcher._load_collections()
+ updated_collections = collection_watcher._get_updated_collections()
+
+ self.assertEquals(len(updated_collections), 0)
+
+ def test_get_updated_collections_returns_some_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+ collection_watcher._load_collections()
+
+ new_collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections_alternate.yml')
+ collection_watcher._collections_path = new_collections_path
+ updated_collections = collection_watcher._get_updated_collections()
+
+ self.assertEquals(len(updated_collections), 1)
+
+ def test_validate_collection(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="/absolute/path",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ collection_watcher._validate_collection(collection)
+
+ def test_validate_collection_with_relative_path(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="relative/path",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ self.assertRaises(RelativePathCollectionError,
collection_watcher._validate_collection, collection)
+
+ def test_validate_collection_with_conflicting_path(self):
+ collections_path = os.path.join(os.path.dirname(__file__),
'/resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(),
Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="/resources/*.nc",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ self.assertRaises(ConflictingPathCollectionError,
collection_watcher._validate_collection, collection)
+
+ def test_collection_callback_is_called(self):
Review comment:
This test failed in my repo. I launched the unit test with pycharm.
Error is:
Failure
Traceback (most recent call last):
File "/Users/loubrieu/opt/anaconda3/lib/python3.7/unittest/case.py", line
59, in testPartExecutor
yield
File "/Users/loubrieu/opt/anaconda3/lib/python3.7/unittest/case.py", line
628, in run
testMethod()
File
"/Users/loubrieu/PycharmProjects/incubator-sdap-nexus-ingestion-manager/collection_manager/tests/services/test_CollectionWatcher.py",
line 143, in test_collection_callback_is_called
self.assert_called_within_timeout(collection_callback, call_count=2)
File
"/Users/loubrieu/PycharmProjects/incubator-sdap-nexus-ingestion-manager/collection_manager/tests/services/test_CollectionWatcher.py",
line 203, in assert_called_within_timeout
raise AssertionError(f"{mock_func} did not reach {call_count} calls
called within {timeout_sec} sec")
AssertionError: <Mock id='140370537938640'> did not reach 2 calls called
within 1.0 sec
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
