jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/399821 )
Change subject: PuppetDB backend: add support for API v4 ...................................................................... PuppetDB backend: add support for API v4 * Add support for PuppetDB API version 4: * Allow to set the API version via configuration. * Default to API v4 as v3 is obsolete. * Use POST for API v4 to overcome GET limits on large queries, fixes T166397. * Bumped minimum version for requests-mock to 1.3.0. Bug: T182575 Change-Id: I5fac12efcd89ea1b42d19aa68deff96df2074424 --- M cumin/backends/puppetdb.py M cumin/tests/unit/backends/test_puppetdb.py A cumin/tests/unit/conftest.py M doc/examples/config.yaml M setup.py 5 files changed, 217 insertions(+), 97 deletions(-) Approvals: Faidon Liambotis: Looks good to me, but someone else must approve jenkins-bot: Verified Volans: Looks good to me, approved diff --git a/cumin/backends/puppetdb.py b/cumin/backends/puppetdb.py index ff4a867..9dc26d8 100644 --- a/cumin/backends/puppetdb.py +++ b/cumin/backends/puppetdb.py @@ -86,7 +86,8 @@ """PuppetDB query builder. The `puppetdb` backend allow to use an existing PuppetDB instance for the hosts selection. - At the moment only PuppetDB v3 API are implemented. + The supported PuppetDB API versions are 3 and 4. It can be specified via the api_version configuration key, if + not configured, the v4 will be used. * Each query part can be composed with the others using boolean operators (``and``, ``or``, ``not``) * Multiple query parts can be grouped together with parentheses (``(``, ``)``). @@ -136,17 +137,13 @@ ``host10[10-42].*.domain or (not F:key1 = value1 and host10*) or (F:key2 > value2 and F:key3 ~ '^value[0-9]+')`` """ - base_url_template = 'https://{host}:{port}/v3/' + base_url_template = 'https://{host}:{port}' """:py:class:`str`: string template in the :py:meth:`str.format` style used to generate the base URL of the PuppetDB server.""" endpoints = {'C': 'resources', 'F': 'nodes', 'O': 'resources', 'P': 'resources', 'R': 'resources'} """:py:class:`dict`: dictionary with the mapping of the available categories in the grammar to the PuppetDB API endpoints.""" - - hosts_keys = {'nodes': 'name', 'resources': 'certname'} - """:py:class:`dict`: dictionary with the mapping of the available endpoints of the PuppetDB API to the field to - query to get the hostname.""" category_prefixes = {'C': '', 'O': 'Role', 'P': 'Profile'} """:py:class:`dict`: dictionary with the mapping of special categories to title prefixes.""" @@ -166,9 +163,19 @@ self.current_group = self.grouped_tokens self._endpoint = None puppetdb_config = self.config.get('puppetdb', {}) - self.url = self.base_url_template.format( + base_url = self.base_url_template.format( host=puppetdb_config.get('host', 'localhost'), port=puppetdb_config.get('port', 443)) + + self.api_version = puppetdb_config.get('api_version', 4) + if self.api_version == 3: + self.url = base_url + '/v3/' + self.hosts_keys = {'nodes': 'name', 'resources': 'certname'} + elif self.api_version == 4: + self.url = base_url + '/pdb/query/v4/' + self.hosts_keys = {'nodes': 'certname', 'resources': 'certname'} + else: + raise InvalidQueryError('Unsupported PuppetDB API version {ver}'.format(ver=self.api_version)) for exception in puppetdb_config.get('urllib3_disable_warnings', []): urllib3.disable_warnings(category=getattr(urllib3.exceptions, exception)) @@ -366,7 +373,7 @@ elif '%' in key: # Querying a specific parameter of the resource - if operator == '~': + if operator == '~' and self.api_version == 3: raise InvalidQueryError('Regex operations are not supported in PuppetDB API v3 for resource parameters') key, param = key.split('%', 1) query_part = ', ["{op}", ["parameter", "{param}"], "{value}"]'.format(op=operator, param=param, value=value) @@ -492,7 +499,11 @@ requests.HTTPError: if the PuppetDB API call fails. """ - resources = requests.get(self.url + self.endpoint, params={'query': query}, verify=True) + if self.api_version == 3: + resources = requests.get(self.url + self.endpoint, params={'query': query}, verify=True) + else: + resources = requests.post(self.url + self.endpoint, json={'query': query}, verify=True) + resources.raise_for_status() return resources.json() diff --git a/cumin/tests/unit/backends/test_puppetdb.py b/cumin/tests/unit/backends/test_puppetdb.py index 7e76bd9..3ed8463 100644 --- a/cumin/tests/unit/backends/test_puppetdb.py +++ b/cumin/tests/unit/backends/test_puppetdb.py @@ -2,7 +2,6 @@ # pylint: disable=invalid-name import mock import pytest -import requests_mock from ClusterShell.NodeSet import NodeSet from requests.exceptions import HTTPError @@ -37,23 +36,37 @@ assert parsed[0].asDict() == hosts -class TestPuppetDBQuery(object): - """PuppetDB backend query test class.""" +class TestPuppetDBQueryV3(object): + """PuppetDB backend query test class for API version 3.""" def setup_method(self, _): - """Setup an instace of PuppetDBQuery for each test.""" - self.query = puppetdb.PuppetDBQuery({}) # pylint: disable=attribute-defined-outside-init + """Set an instance of PuppetDBQuery for each test.""" + config = {'puppetdb': {'api_version': 3}} + self.query = puppetdb.PuppetDBQuery(config) # pylint: disable=attribute-defined-outside-init def test_instantiation(self): """An instance of PuppetDBQuery should be an instance of BaseQuery.""" assert isinstance(self.query, BaseQuery) assert self.query.url == 'https://localhost:443/v3/' + +class TestPuppetDBQueryV4(object): + """PuppetDB backend query test class for API version 4.""" + + def setup_method(self, _): + """Set an instance of PuppetDBQuery for each test.""" + self.query = puppetdb.PuppetDBQuery({}) # pylint: disable=attribute-defined-outside-init + + def test_instantiation(self): + """An instance of PuppetDBQuery should be an instance of BaseQuery.""" + assert isinstance(self.query, BaseQuery) + assert self.query.url == 'https://localhost:443/pdb/query/v4/' + def test_endpoint_getter(self): """Access to endpoint property should return nodes by default.""" assert self.query.endpoint == 'nodes' - @pytest.mark.parametrize('endpoint', puppetdb.PuppetDBQuery.hosts_keys) + @pytest.mark.parametrize('endpoint', set(puppetdb.PuppetDBQuery.endpoints.values())) def test_endpoint_setter_valid(self, endpoint): """Setting the endpoint property should accept valid values.""" self.query.endpoint = endpoint @@ -81,12 +94,71 @@ self.query.endpoint = 'resources' +def test_puppetdb_query_init_invalid(): + """Instantiating PuppetDBQuery with an unsupported API version should raise InvalidQueryError.""" + with pytest.raises(InvalidQueryError, match='Unsupported PuppetDB API version'): + puppetdb.PuppetDBQuery({'puppetdb': {'api_version': 99}}) + + @mock.patch.object(puppetdb.PuppetDBQuery, '_api_call') -class TestPuppetDBQueryBuild(object): - """PuppetDB backend query build test class.""" +class TestPuppetDBQueryBuildV3(object): + """PuppetDB backend API v3 query build test class.""" def setup_method(self, _): - """Setup an instace of PuppetDBQuery for each test.""" + """Set an instace of PuppetDBQuery for each test.""" + config = {'puppetdb': {'api_version': 3}} + self.query = puppetdb.PuppetDBQuery(config) # pylint: disable=attribute-defined-outside-init + + def test_add_category_resource_parameter_regex(self, mocked_api_call): + """A resource's parameter query with a regex should raise InvalidQueryError.""" + with pytest.raises(InvalidQueryError, match='Regex operations are not supported in PuppetDB'): + self.query.execute('R:resource%param ~ value.*') + assert not mocked_api_call.called + + def test_add_hosts(self, mocked_api_call): + """A host query should add the proper query token to the current_group.""" + # No hosts + self.query.execute('host1!host1') + mocked_api_call.assert_called_with('') + # Single host + self.query.execute('host') + mocked_api_call.assert_called_with('["or", ["=", "name", "host"]]') + # Multiple hosts + self.query.execute('host[1-2]') + mocked_api_call.assert_called_with('["or", ["=", "name", "host1"], ["=", "name", "host2"]]') + # Negated query + self.query.execute('not host[1-2]') + mocked_api_call.assert_called_with('["not", ["or", ["=", "name", "host1"], ["=", "name", "host2"]]]') + # Globbing hosts + self.query.execute('host1*.domain') + mocked_api_call.assert_called_with(r'["or", ["~", "name", "^host1.*\\.domain$"]]') + + def test_and(self, mocked_api_call): + """A query with 'and' should set the boolean property to the current group to 'and'.""" + self.query.execute('host1 and host2') + assert self.query.current_group['bool'] == 'and' + mocked_api_call.assert_called_with('["and", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]]]') + + def test_or(self, mocked_api_call): + """A query with 'or' should set the boolean property to the current group to 'or'.""" + self.query.execute('host1 or host2') + assert self.query.current_group['bool'] == 'or' + mocked_api_call.assert_called_with('["or", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]]]') + + def test_and_and(self, mocked_api_call): + """A query with 'and' and 'and' should set the boolean property to the current group to 'and'.""" + self.query.execute('host1 and host2 and host3') + assert self.query.current_group['bool'] == 'and' + mocked_api_call.assert_called_with( + '["and", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]], ["or", ["=", "name", "host3"]]]') + + +@mock.patch.object(puppetdb.PuppetDBQuery, '_api_call') +class TestPuppetDBQueryBuildV4(object): + """PuppetDB backend API v4 query build test class.""" + + def setup_method(self, _): + """Set an instace of PuppetDBQuery for each test.""" self.query = puppetdb.PuppetDBQuery({}) # pylint: disable=attribute-defined-outside-init def test_add_category_fact(self, mocked_api_call): @@ -142,10 +214,10 @@ '["and", ["=", "type", "Resource"], ["=", ["parameter", "param"], "value"]]') def test_add_category_resource_parameter_regex(self, mocked_api_call): - """A resource's parameter query with a regex should raise InvalidQueryError.""" - with pytest.raises(InvalidQueryError, match='Regex operations are not supported in PuppetDB'): - self.query.execute('R:resource%param ~ value.*') - assert not mocked_api_call.called + """A resource's parameter query with a regex should add the propery query token to the object.""" + self.query.execute('R:resource%param ~ value.*') + mocked_api_call.assert_called_with( + '["and", ["=", "type", "Resource"], ["~", ["parameter", "param"], "value.*"]]') def test_add_category_resource_field(self, mocked_api_call): """A resource's field query should add the proper query token to the current_group.""" @@ -302,28 +374,30 @@ mocked_api_call.assert_called_with('') # Single host self.query.execute('host') - mocked_api_call.assert_called_with('["or", ["=", "name", "host"]]') + mocked_api_call.assert_called_with('["or", ["=", "certname", "host"]]') # Multiple hosts self.query.execute('host[1-2]') - mocked_api_call.assert_called_with('["or", ["=", "name", "host1"], ["=", "name", "host2"]]') + mocked_api_call.assert_called_with('["or", ["=", "certname", "host1"], ["=", "certname", "host2"]]') # Negated query self.query.execute('not host[1-2]') - mocked_api_call.assert_called_with('["not", ["or", ["=", "name", "host1"], ["=", "name", "host2"]]]') + mocked_api_call.assert_called_with('["not", ["or", ["=", "certname", "host1"], ["=", "certname", "host2"]]]') # Globbing hosts self.query.execute('host1*.domain') - mocked_api_call.assert_called_with(r'["or", ["~", "name", "^host1.*\\.domain$"]]') + mocked_api_call.assert_called_with(r'["or", ["~", "certname", "^host1.*\\.domain$"]]') def test_and(self, mocked_api_call): """A query with 'and' should set the boolean property to the current group to 'and'.""" self.query.execute('host1 and host2') assert self.query.current_group['bool'] == 'and' - mocked_api_call.assert_called_with('["and", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]]]') + mocked_api_call.assert_called_with( + '["and", ["or", ["=", "certname", "host1"]], ["or", ["=", "certname", "host2"]]]') def test_or(self, mocked_api_call): """A query with 'or' should set the boolean property to the current group to 'or'.""" self.query.execute('host1 or host2') assert self.query.current_group['bool'] == 'or' - mocked_api_call.assert_called_with('["or", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]]]') + mocked_api_call.assert_called_with( + '["or", ["or", ["=", "certname", "host1"]], ["or", ["=", "certname", "host2"]]]') def test_and_or(self, mocked_api_call): """A query with 'and' and 'or' in the same group should raise InvalidQueryError.""" @@ -336,84 +410,60 @@ self.query.execute('host1 and host2 and host3') assert self.query.current_group['bool'] == 'and' mocked_api_call.assert_called_with( - '["and", ["or", ["=", "name", "host1"]], ["or", ["=", "name", "host2"]], ["or", ["=", "name", "host3"]]]') + ('["and", ["or", ["=", "certname", "host1"]], ["or", ["=", "certname", "host2"]], ' + '["or", ["=", "certname", "host3"]]]')) -@requests_mock.Mocker() -class TestPuppetDBQueryExecute(object): - """PuppetDBQuery test execute() method class.""" +def test_nodes_endpoint(query_requests): + """Calling execute() with a query that goes to the nodes endpoint should return the list of hosts.""" + hosts = query_requests[0].execute('nodes_host[1-2]') + assert hosts == NodeSet('nodes_host[1-2]') + assert query_requests[1].call_count == 1 - def setup_method(self, _): - """Setup an instace of PuppetDBQuery for each test.""" - # pylint: disable=attribute-defined-outside-init - self.query = puppetdb.PuppetDBQuery({'puppetdb': {'urllib3_disable_warnings': ['SubjectAltNameWarning']}}) - def _register_requests_uris(self, requests): - """Setup the requests library mock for each test.""" - # Register a requests valid response for each endpoint - for endpoint, key in self.query.hosts_keys.items(): - requests.register_uri('GET', self.query.url + endpoint + '?query=', status_code=200, json=[ - {key: endpoint + '_host1', 'key': 'value1'}, {key: endpoint + '_host2', 'key': 'value2'}]) +def test_resources_endpoint(query_requests): + """Calling execute() with a query that goes to the resources endpoint should return the list of hosts.""" + hosts = query_requests[0].execute('R:Class = value') + assert hosts == NodeSet('resources_host[1-2]') + assert query_requests[1].call_count == 1 - # Register a requests response for a non matching query - requests.register_uri( - 'GET', self.query.url + self.query.endpoints['F'] + '?query=["or", ["=", "name", "non_existent_host"]]', - status_code=200, json=[], complete_qs=True) - # Register a requests response for an invalid query - requests.register_uri( - 'GET', self.query.url + self.query.endpoints['F'] + '?query=["or", ["=", "name", "invalid_query"]]', - status_code=400, complete_qs=True) - def test_nodes_endpoint(self, requests): - """Calling execute() with a query that goes to the nodes endpoint should return the list of hosts.""" - self._register_requests_uris(requests) - hosts = self.query.execute('nodes_host[1-2]') - assert hosts == NodeSet('nodes_host[1-2]') - assert requests.call_count == 1 +def test_with_boolean_operator(query_requests): + """Calling execute() with a query with a boolean operator should return the list of hosts.""" + hosts = query_requests[0].execute('nodes_host1 or nodes_host2') + assert hosts == NodeSet('nodes_host[1-2]') + assert query_requests[1].call_count == 1 - def test_resources_endpoint(self, requests): - """Calling execute() with a query that goes to the resources endpoint should return the list of hosts.""" - self._register_requests_uris(requests) - hosts = self.query.execute('R:Class = value') - assert hosts == NodeSet('resources_host[1-2]') - assert requests.call_count == 1 - def test_with_boolean_operator(self, requests): - """Calling execute() with a query with a boolean operator should return the list of hosts.""" - self._register_requests_uris(requests) - hosts = self.query.execute('nodes_host1 or nodes_host2') - assert hosts == NodeSet('nodes_host[1-2]') - assert requests.call_count == 1 +def test_with_subgroup(query_requests): + """Calling execute() with a query with a subgroup return the list of hosts.""" + hosts = query_requests[0].execute('(nodes_host1 or nodes_host2)') + assert hosts == NodeSet('nodes_host[1-2]') + assert query_requests[1].call_count == 1 - def test_with_subgroup(self, requests): - """Calling execute() with a query with a subgroup return the list of hosts.""" - self._register_requests_uris(requests) - hosts = self.query.execute('(nodes_host1 or nodes_host2)') - assert hosts == NodeSet('nodes_host[1-2]') - assert requests.call_count == 1 - def test_empty(self, requests): - """Calling execute() with a query that return no hosts should return an empty list.""" - self._register_requests_uris(requests) - hosts = self.query.execute('non_existent_host') - assert hosts == NodeSet() - assert requests.call_count == 1 +def test_empty(query_requests): + """Calling execute() with a query that return no hosts should return an empty list.""" + hosts = query_requests[0].execute('non_existent_host') + assert hosts == NodeSet() + assert query_requests[1].call_count == 1 - def test_error(self, requests): - """Calling execute() if the request fails it should raise the requests exception.""" - self._register_requests_uris(requests) - with pytest.raises(HTTPError): - self.query.execute('invalid_query') - assert requests.call_count == 1 - def test_complex_query(self, requests): - """Calling execute() with a complex query should return the exptected structure.""" - category = 'R' - endpoint = self.query.endpoints[category] - key = self.query.hosts_keys[endpoint] - requests.register_uri('GET', self.query.url + endpoint + '?query=', status_code=200, json=[ - {key: endpoint + '_host1', 'key': 'value1'}, {key: endpoint + '_host2', 'key': 'value2'}]) +def test_error(query_requests): + """Calling execute() if the request fails it should raise the requests exception.""" + with pytest.raises(HTTPError): + query_requests[0].execute('invalid_query') + assert query_requests[1].call_count == 1 - hosts = self.query.execute('(resources_host1 or resources_host2) and R:Class = MyClass') - assert hosts == NodeSet('resources_host[1-2]') - assert requests.call_count == 1 + +def test_complex_query(query_requests): + """Calling execute() with a complex query should return the exptected structure.""" + category = 'R' + endpoint = query_requests[0].endpoints[category] + key = query_requests[0].hosts_keys[endpoint] + query_requests[1].register_uri('GET', query_requests[0].url + endpoint + '?query=', status_code=200, json=[ + {key: endpoint + '_host1', 'key': 'value1'}, {key: endpoint + '_host2', 'key': 'value2'}]) + + hosts = query_requests[0].execute('(resources_host1 or resources_host2) and R:Class = MyClass') + assert hosts == NodeSet('resources_host[1-2]') + assert query_requests[1].call_count == 1 diff --git a/cumin/tests/unit/conftest.py b/cumin/tests/unit/conftest.py new file mode 100644 index 0000000..36b7a5d --- /dev/null +++ b/cumin/tests/unit/conftest.py @@ -0,0 +1,58 @@ +"""Pytest customization for unit tests.""" +import pytest +import requests_mock + +from cumin.backends import puppetdb + + +def _requests_matcher_non_existent(request): + return request.json() == {'query': '["or", ["=", "certname", "non_existent_host"]]'} + + +def _requests_matcher_invalid(request): + return request.json() == {'query': '["or", ["=", "certname", "invalid_query"]]'} + + +@pytest.fixture() +def mocked_requests(): + """Set mocked requests fixture.""" + with requests_mock.Mocker() as mocker: + yield mocker + + +@pytest.fixture(params=(3, 4)) +def query_requests(request, mocked_requests): # pylint: disable=redefined-outer-name + """Set the requests library mock for each test and PuppetDB API version.""" + if request.param == 3: # PuppetDB API v3 + query = puppetdb.PuppetDBQuery( + {'puppetdb': {'api_version': 3, 'urllib3_disable_warnings': ['SubjectAltNameWarning']}}) + for endpoint, key in query.hosts_keys.items(): + mocked_requests.register_uri('GET', query.url + endpoint + '?query=', status_code=200, json=[ + {key: endpoint + '_host1', 'key': 'value1'}, {key: endpoint + '_host2', 'key': 'value2'}]) + + # Register a requests response for a non matching query + mocked_requests.register_uri( + 'GET', query.url + query.endpoints['F'] + '?query=["or", ["=", "name", "non_existent_host"]]', + status_code=200, json=[], complete_qs=True) + # Register a requests response for an invalid query + mocked_requests.register_uri( + 'GET', query.url + query.endpoints['F'] + '?query=["or", ["=", "name", "invalid_query"]]', + status_code=400, complete_qs=True) + + elif request.param == 4: # PuppetDB API v4 + query = puppetdb.PuppetDBQuery({}) + for endpoint, key in query.hosts_keys.items(): + mocked_requests.register_uri( + 'POST', query.url + endpoint, status_code=200, complete_qs=True, + json=[{key: endpoint + '_host1', 'key': 'value1'}, {key: endpoint + '_host2', 'key': 'value2'}]) + + # Register a requests response for a non matching query + mocked_requests.register_uri( + 'POST', query.url + query.endpoints['F'], status_code=200, json=[], complete_qs=True, + additional_matcher=_requests_matcher_non_existent) + # Register a requests response for an invalid query + mocked_requests.register_uri( + 'POST', query.url + query.endpoints['F'], status_code=400, complete_qs=True, + additional_matcher=_requests_matcher_invalid) + + return query, mocked_requests diff --git a/doc/examples/config.yaml b/doc/examples/config.yaml index 0db6974..b546b49 100644 --- a/doc/examples/config.yaml +++ b/doc/examples/config.yaml @@ -16,6 +16,7 @@ puppetdb: host: puppetdb.local port: 443 + api_version: 4 # Supported versions are v3 and v4. If not specified, v4 will be used. urllib3_disable_warnings: # List of Python urllib3 exceptions to ignore - SubjectAltNameWarning diff --git a/setup.py b/setup.py index 66f2d53..53be76c 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'pytest-cov>=1.8.0', 'pytest-xdist>=1.15.0', 'pytest>=3.0.3', - 'requests-mock>=0.7.0', + 'requests-mock>=1.3.0', 'sphinx_rtd_theme>=0.1.6', 'sphinx-argparse>=0.1.15', 'Sphinx>=1.4.9', -- To view, visit https://gerrit.wikimedia.org/r/399821 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5fac12efcd89ea1b42d19aa68deff96df2074424 Gerrit-PatchSet: 6 Gerrit-Project: operations/software/cumin Gerrit-Branch: master Gerrit-Owner: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Herron <kher...@wikimedia.org> Gerrit-Reviewer: Volans <rcocci...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits