AMBARI-14989. Inconsistent HIVE CBO setting presentation. (Jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b9a35f1e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b9a35f1e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b9a35f1e Branch: refs/heads/branch-dev-patch-upgrade Commit: b9a35f1ea6201c7f7c45a9f1463c0802504ee4a5 Parents: 6270fe3 Author: Jaimin Jetly <jai...@hortonworks.com> Authored: Thu Feb 11 15:02:09 2016 +0530 Committer: Jaimin Jetly <jai...@hortonworks.com> Committed: Thu Feb 11 15:05:22 2016 +0530 ---------------------------------------------------------------------- .../services/HIVE/configuration/hive-env.xml | 20 --------- .../services/HIVE/configuration/hive-site.xml | 14 ------ .../stacks/HDP/2.2/services/stack_advisor.py | 13 +++--- .../services/HIVE/configuration/hive-env.xml | 20 --------- .../services/HIVE/configuration/hive-site.xml | 8 ---- .../stacks/HDPWIN/2.2/services/stack_advisor.py | 13 +++--- .../2.1/configs/hive-metastore-upgrade.json | 3 +- .../stacks/2.2/common/test_stack_advisor.py | 12 +++-- .../2.3/common/services-sparkts-hive.json | 47 -------------------- .../stacks/2.3/common/test_stack_advisor.py | 5 +-- 10 files changed, 18 insertions(+), 137 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml index 87adb4b..471af6e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -130,26 +130,6 @@ export METASTORE_PORT={{hive_metastore_port}} </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 96efea6..25bb468 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -471,12 +471,6 @@ limitations under the License. <name>hive.cbo.enable</name> <value>true</value> <description>Flag to control enabling Cost Based Optimizations using Calcite framework.</description> - <depends-on> - <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - </depends-on> </property> <property> @@ -820,10 +814,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -855,10 +845,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9e43865..dc968cc 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -353,14 +353,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml index 0efd749..8dde3fd 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml @@ -77,26 +77,6 @@ </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml index 6ef7f32..be7c943 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml @@ -789,10 +789,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -824,10 +820,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index afdb7cf..1ed4ce0 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -362,14 +362,11 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json index 7829f05..0fbc282 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json @@ -77,8 +77,7 @@ "hive_user": "hive", "hive_database": "New MySQL Database", "hive.heapsize": "703", - "hcat_pid_dir": "/var/run/webhcat", - "cost_based_optimizer": "On", + "hcat_pid_dir": "/var/run/webhcat", "webhcat_user": "hcat", "hive_security_authorization": "None", "hive_exec_orc_storage_strategy": "SPEED", http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 7d81423..d2497fd 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1019,7 +1019,6 @@ class TestHDP22StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -1033,7 +1032,6 @@ class TestHDP22StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1176,7 +1174,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1251,7 +1250,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1315,10 +1315,8 @@ class TestHDP22StackAdvisor(TestCase): #test recommendations services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] = "false" services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - services["changed-configurations"] = [{"type": "hive-site", "name": "hive.cbo.enable"}, - {"type": "hive-env", "name": "hive_security_authorization"}] + services["changed-configurations"] = [{"type": "hive-env", "name": "hive_security_authorization"}] expected["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - expected["hive-site"]["properties"]["hive.cbo.enable"] = "false" expected["hive-site"]["properties"]["hive.stats.fetch.partition.stats"]="false" expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false" expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true" http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json index 51ae36a..20a672e 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json @@ -2402,47 +2402,6 @@ }, "dependencies" : [ ] }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer", - "StackConfigurations" : { - "property_depends_on" : [ ], - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3", - "type" : "hive-env.xml" - }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.cbo.enable", - "StackConfigurationDependency" : { - "dependency_name" : "hive.cbo.enable", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.column.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.column.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.partition.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.partition.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] - }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/datanucleus.autoCreateSchema", "StackConfigurations" : { "property_depends_on" : [ ], @@ -4199,9 +4158,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.column.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], @@ -4216,9 +4172,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.partition.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/b9a35f1e/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 72e7718..545a2b5 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -882,7 +882,6 @@ class TestHDP23StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -896,7 +895,6 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1040,7 +1038,8 @@ class TestHDP23StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": {