This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 81305f205849f20fe3d31a142b10aee16d5c46de Author: Anurag Mantripragada <anuragmantr...@gmail.com> AuthorDate: Thu Feb 7 14:18:26 2019 -0800 IMPALA-8163: Add banner showing local catalog mode on web UI when enabled. Added a banner on coordinator web UI for visibility of local catalog mode. Testing: 1. Added a test in test_local_catalog to scrape webpages and search for strings. Tested banner appears when local catalog mode is enabled. 2. Refactored parts of test_local_catalog into TestObservability class. Change-Id: Iaeb2c7b20742a2630d9509da9b629d5f45c55207 Reviewed-on: http://gerrit.cloudera.org:8080/12413 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- be/src/util/default-path-handlers.cc | 3 + tests/custom_cluster/test_local_catalog.py | 128 +++++++++++++++-------------- www/root.tmpl | 3 +- 3 files changed, 72 insertions(+), 62 deletions(-) diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc index 4458b83..6beb148 100644 --- a/be/src/util/default-path-handlers.cc +++ b/be/src/util/default-path-handlers.cc @@ -48,6 +48,7 @@ using namespace rapidjson; using namespace strings; DECLARE_bool(enable_process_lifetime_heap_profiling); +DECLARE_bool(use_local_catalog); DEFINE_int64(web_log_bytes, 1024 * 1024, "The maximum number of bytes to display on the debug webserver's log page"); @@ -285,6 +286,8 @@ void RootHandler(const Webserver::ArgumentMap& args, Document* document) { document->AddMember("impala_server_mode", true, document->GetAllocator()); document->AddMember("is_coordinator", impala_server->IsCoordinator(), document->GetAllocator()); + document->AddMember("use_local_catalog", FLAGS_use_local_catalog, + document->GetAllocator()); document->AddMember("is_executor", impala_server->IsExecutor(), document->GetAllocator()); bool is_quiescing = impala_server->IsShuttingDown(); diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py index f9399d7..6173bde 100644 --- a/tests/custom_cluster/test_local_catalog.py +++ b/tests/custom_cluster/test_local_catalog.py @@ -29,21 +29,6 @@ RETRY_PROFILE_MSG = 'Retried query planning due to inconsistent metadata' class TestCompactCatalogUpdates(CustomClusterTestSuite): - def get_catalog_cache_metrics(self, impalad): - """ Returns catalog cache metrics as a dict by scraping the json metrics page on the - given impalad""" - child_groups =\ - impalad.service.get_debug_webpage_json('metrics')['metric_group']['child_groups'] - for group in child_groups: - if group['name'] != 'impala-server': continue - # Filter catalog cache metrics. - for child_group in group['child_groups']: - if child_group['name'] != 'catalog': continue - metrics_data = [(metric['name'], metric['value']) - for metric in child_group['metrics'] if 'catalog.cache' in metric['name']] - return dict(metrics_data) - assert False, "Catalog cache metrics not found in %s" % child_groups - @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args="--use_local_catalog=true", @@ -167,52 +152,6 @@ class TestCompactCatalogUpdates(CustomClusterTestSuite): finally: client.close() - @pytest.mark.execute_serially - @CustomClusterTestSuite.with_args( - impalad_args="--use_local_catalog=true", - catalogd_args="--catalog_topic_mode=minimal") - def test_cache_metrics(self, unique_database): - """ - Test that profile output includes impalad local cache metrics. Also verifies that - the daemon level metrics are updated between query runs. - """ - try: - impalad = self.cluster.impalads[0] - client = impalad.service.create_beeswax_client() - cache_hit_rate_metric_key = "catalog.cache.hit-rate" - cache_miss_rate_metric_key = "catalog.cache.miss-rate" - cache_hit_count_metric_key = "catalog.cache.hit-count" - cache_request_count_metric_key = "catalog.cache.request-count" - cache_request_count_prev_run = 0 - cache_hit_count_prev_run = 0 - test_table_name = "%s.test_cache_metrics_test_tbl" % unique_database - # A mix of queries of various types. - queries_to_test = ["select count(*) from functional.alltypes", - "explain select count(*) from functional.alltypes", - "create table %s (a int)" % test_table_name, - "drop table %s" % test_table_name] - for _ in xrange(0, 10): - for query in queries_to_test: - ret = self.execute_query_expect_success(client, query) - assert ret.runtime_profile.count("Frontend:") == 1 - assert ret.runtime_profile.count("CatalogFetch") > 1 - cache_metrics = self.get_catalog_cache_metrics(impalad) - cache_hit_rate = cache_metrics[cache_hit_rate_metric_key] - cache_miss_rate = cache_metrics[cache_miss_rate_metric_key] - cache_hit_count = cache_metrics[cache_hit_count_metric_key] - cache_request_count = cache_metrics[cache_request_count_metric_key] - assert cache_hit_rate > 0.0 and cache_hit_rate < 1.0 - assert cache_miss_rate > 0.0 and cache_miss_rate < 1.0 - assert cache_hit_count > cache_hit_count_prev_run,\ - "%s not updated between two query runs, query - %s"\ - % (cache_hit_count_metric_key, query) - assert cache_request_count > cache_request_count_prev_run,\ - "%s not updated betweeen two query runs, query - %s"\ - % (cache_request_count_metric_key, query) - cache_hit_count_prev_run = cache_hit_count - cache_request_count_prev_run = cache_request_count - finally: - client.close() class TestLocalCatalogRetries(CustomClusterTestSuite): @@ -373,3 +312,70 @@ class TestLocalCatalogRetries(CustomClusterTestSuite): finally: client1.close() client2.close() + + +class TestObservability(CustomClusterTestSuite): + + def get_catalog_cache_metrics(self, impalad): + """ Returns catalog cache metrics as a dict by scraping the json metrics page on the + given impalad""" + child_groups =\ + impalad.service.get_debug_webpage_json('metrics')['metric_group']['child_groups'] + for group in child_groups: + if group['name'] != 'impala-server': continue + # Filter catalog cache metrics. + for child_group in group['child_groups']: + if child_group['name'] != 'catalog': continue + metrics_data = [(metric['name'], metric['value']) + for metric in child_group['metrics'] if 'catalog.cache' in metric['name']] + return dict(metrics_data) + assert False, "Catalog cache metrics not found in %s" % child_groups + + @pytest.mark.execute_serially + @CustomClusterTestSuite.with_args( + impalad_args="--use_local_catalog=true", + catalogd_args="--catalog_topic_mode=minimal") + def test_cache_metrics(self, unique_database): + """ + Test that profile output includes impalad local cache metrics. Also verifies that + the daemon level metrics are updated between query runs. + """ + try: + impalad = self.cluster.impalads[0] + # Make sure local catalog mode is enabled and visible on web UI. + assert '(Local Catalog Mode)' in impalad.service.read_debug_webpage('/') + client = impalad.service.create_beeswax_client() + cache_hit_rate_metric_key = "catalog.cache.hit-rate" + cache_miss_rate_metric_key = "catalog.cache.miss-rate" + cache_hit_count_metric_key = "catalog.cache.hit-count" + cache_request_count_metric_key = "catalog.cache.request-count" + cache_request_count_prev_run = 0 + cache_hit_count_prev_run = 0 + test_table_name = "%s.test_cache_metrics_test_tbl" % unique_database + # A mix of queries of various types. + queries_to_test = ["select count(*) from functional.alltypes", + "explain select count(*) from functional.alltypes", + "create table %s (a int)" % test_table_name, + "drop table %s" % test_table_name] + for _ in xrange(0, 10): + for query in queries_to_test: + ret = self.execute_query_expect_success(client, query) + assert ret.runtime_profile.count("Frontend:") == 1 + assert ret.runtime_profile.count("CatalogFetch") > 1 + cache_metrics = self.get_catalog_cache_metrics(impalad) + cache_hit_rate = cache_metrics[cache_hit_rate_metric_key] + cache_miss_rate = cache_metrics[cache_miss_rate_metric_key] + cache_hit_count = cache_metrics[cache_hit_count_metric_key] + cache_request_count = cache_metrics[cache_request_count_metric_key] + assert cache_hit_rate > 0.0 and cache_hit_rate < 1.0 + assert cache_miss_rate > 0.0 and cache_miss_rate < 1.0 + assert cache_hit_count > cache_hit_count_prev_run,\ + "%s not updated between two query runs, query - %s"\ + % (cache_hit_count_metric_key, query) + assert cache_request_count > cache_request_count_prev_run,\ + "%s not updated betweeen two query runs, query - %s"\ + % (cache_request_count_metric_key, query) + cache_hit_count_prev_run = cache_hit_count + cache_request_count_prev_run = cache_request_count + finally: + client.close() diff --git a/www/root.tmpl b/www/root.tmpl index a33df9b..ef42be6 100644 --- a/www/root.tmpl +++ b/www/root.tmpl @@ -27,7 +27,8 @@ under the License. {{/is_quiescing}} {{?impala_server_mode}} - <h2>Impala Server Mode: {{?is_coordinator}}Coordinator{{/is_coordinator}} + <h2>Impala Server Mode: {{?is_coordinator}}Coordinator{{?use_local_catalog}} + (Local Catalog Mode){{/use_local_catalog}}{{/is_coordinator}} {{?is_executor}}Executor{{/is_executor}}</h2> {{/impala_server_mode}}